Analyses of learning via shared features experiments

# disable scientific notation
options(scipen = 999) 

# dependencies
library(tidyverse)
library(effsize)
library(foreign)
library(timesavers)
library(BayesFactor)
library(weights)
library(sjPlot)
library(nnet)
library(metafor)
library(car)
library(lsr)
library(knitr)
library(kableExtra)

Get data

data_exp_1 <- read_csv("../experiment 1/data/processed/data_for_analysis.csv")
data_exp_2 <- read_csv("../experiment 2/data/processed/data_for_analysis.csv")
data_exp_3 <- read_csv("../experiment 3/data/processed/data_for_analysis.csv")
data_exp_4 <- read_csv("../experiment 4/data/processed/data_for_analysis.csv")
data_exp_5 <- read_csv("../experiment 5/data/processed/data_for_analysis.csv")
data_exp_6 <- read_csv("../experiment 6/data/processed/data_for_analysis.csv")
data_exp_7 <- read_csv("../experiment 7/data/processed/data_for_analysis.csv")
data_exp_8 <- read_csv("../experiment 8/data/processed/data_for_analysis.csv")
data_exp_9 <- read_csv("../experiment 9/data/processed/data_for_analysis.csv")

# subsets of conditions
data_exp_8nri <- data_exp_8 %>%
  filter(Bag_Matching == "Shared_Feature_Bag_Location_Condition_1")

data_exp_8ri <- data_exp_8 %>%
  filter(Bag_Matching == "Shared_Feature_Bag_Random_Condition_3")

Analytic strategy

# analysis workflow. is applied witin all studies ---------------

# between groups comparisons analysis workflow ------
t_test_analyses <- function(data) {
  
  # summary stats
  summary_stats <- data %>%
    group_by(IV) %>%
    dplyr::summarize(mean = mean(DV),
                     sd = sd(DV)) %>%
    round_df(2)
  
  # t test
  t_test <- t.test(formula = DV ~ IV,
                   alternative = "two.sided",
                   paired = FALSE,
                   data = data)
  
  # effect size
  # formula based cohen's d seems to be invariant to ordering of the levels, can't control whether the direction 
  # cohens_d <- effsize::cohen.d(formula = DV ~ IV,
  #                              paired = FALSE,
  #                              data = data)
  cohens_d <- effsize::cohen.d(data$DV[data$IV == "CS1_Positive_CS2_Negative"],
                               data$DV[data$IV == "CS2_Positive_CS1_Negative"],
                               paired = FALSE)
  
  # printing
  t_test_p <- t_test$p.value[[1]]
  t_test_p_apa <- ifelse(t_test_p < 0.001, "< .001", 
                         ifelse(t_test_p < 0.01,
                                paste("= ", rd(t_test_p, 3), sep = ""),  # rd() rounds, converts to string, and removes the leading 0.
                                paste("= ", rd(t_test_p, 2), sep = "")))
  
  t_test_output <- paste("t(", format(round(t_test$parameter[[1]], 2), nsmall = 2), 
                         ") = ", format(round(t_test$statistic[[1]], 2), nsmall = 2), 
                         ", p ", t_test_p_apa, 
                         ", d = ", format(round(cohens_d$estimate[[1]], 2), nsmall = 2), 
                         ", 95% CI = [", format(round(cohens_d$conf.int[["lower"]], 2), nsmall = 2), ", ", 
                         format(round(cohens_d$conf.int[["upper"]], 2), nsmall = 2), "]", 
                         sep = "")
  
  # bayes factor t test
  t_test_bf <- ttestBF(formula = DV ~ IV,
                       data = data)
  
  bf10 <- round(exp(t_test_bf@bayesFactor$bf), 2)
  
  return(list(summary_stats = summary_stats,
              t_test = t_test,
              cohens_d = cohens_d,
              t_test_output = t_test_output,
              t_test_p = t_test_p,
              t_test_bf = t_test_bf,
              bf10 = bf10))
}

# multinominal analysis workflow ------
multinominal_analysis <- function(data) {
  
  # fit model
  fit <- multinom(DV ~ IV, 
                  data = data)
  
  # calculate p values
  z <- summary(fit)$coefficients/summary(fit)$standard.errors
  p <- (1 - pnorm(abs(z), 0, 1)) * 2
  
  pvalues <- p %>%
    as.data.frame() %>%
    rownames_to_column(var = "var") %>%
    
    rename(p = "IVCS2_Positive_CS1_Negative") %>%
    mutate(p = apa_p_value(p)) %>%
    select(p)
  
  # calc odds ratios and CIs
  coefficients <- as.data.frame(summary(fit)$coefficients) %>%
    rownames_to_column(var = "var") %>%
    rename(logOR = "IVCS2_Positive_CS1_Negative") %>%
    select(var, logOR)
  
  se <- as.data.frame(summary(fit)$standard.errors) %>%
    rownames_to_column(var = "var") %>%
    rename(logOR_se = "IVCS2_Positive_CS1_Negative") %>%
    select(logOR_se)
  
  # combine
  results <- cbind(coefficients, se) %>%
    mutate(OR = round(exp(logOR), 2),
           OR_ci_lwr = round(exp(logOR - logOR_se*1.96), 2),
           OR_ci_upr = round(exp(logOR + logOR_se*1.96), 2)) %>%
    cbind(pvalues) 
  
  return(results)
}

# meta analysis workflow ------
meta_analysis_workflow <- function(data, 
                                   effect_size_label, 
                                   reference_line, 
                                   exp_estimates = FALSE,
                                   plot = TRUE) {
  require(timesavers)
  require(tidyverse)
  require(metafor)
  
  # fit random effects model 
  fitted_model <- 
    rma(yi = yi, 
        sei = sei,
        data = data,
        slab = Experiment)
  
  p_value <- apa_p_value(fitted_model$pval)
  
  # model predictions
  meta_analysis_results <-
    predict(fitted_model, digits = 5) %>%
    as.data.frame() %>%
    gather() %>%
    round_df(2) %>%
    dplyr::rename(metric = key,
                  estimate = value) %>%
    mutate(metric = dplyr::recode(metric,
                                  "pred" = paste0("Meta analysed ", effect_size_label),
                                  "ci.lb" = "95% CI lower",
                                  "ci.ub" = "95% CI upper",
                                  "cr.lb" = "95% CR lower",
                                  "cr.ub" = "95% CR upper"))
  
  # source_valence_conditionally exponentiate estimates if converting log odds to odds ratios
  if(exp_estimates == TRUE) {
    meta_analysis_results <- meta_analysis_results %>%
      mutate(estimate = round(exp(estimate), 2))
  }
  
  meta_analysis_results <- rbind(meta_analysis_results,
                                 data.frame(metric = "p",
                                            estimate = p_value))
  
  # summarize results
  meta_analysis_results_text <- 
    paste0("k = ", fitted_model$k, ", ", 
           effect_size_label, " = ", meta_analysis_results$estimate[1],
           # dynamic indexing of some values as different models return variables in different locations, but relative location is reliable
           ", 95% CI = [", meta_analysis_results$estimate[length(meta_analysis_results$estimate)-4],  
           ", ", meta_analysis_results$estimate[length(meta_analysis_results$estimate)-3], 
           "], 95% CR = [", meta_analysis_results$estimate[length(meta_analysis_results$estimate)-2], 
           ", ", meta_analysis_results$estimate[length(meta_analysis_results$estimate)-1],
           "], p ", meta_analysis_results$estimate[length(meta_analysis_results$estimate)])
  
  heterogeneity_test_results_text <- 
    paste0("Q(df = ",    fitted_model$k - 1, ") = ", round(fitted_model$QE, 2), 
           ", p ",     ifelse(fitted_model$pval < 0.001, "< .001", as.character(paste("=", round(fitted_model$pval, 3)))),
           ", tau^2 = ", round(fitted_model$tau2, 2), 
           ", I^2 = ",   round(fitted_model$I2, 2), "%",
           ", H^2 = ",   round(fitted_model$H2, 2))
  
  # forest plot
  if (exp_estimates == TRUE) {
    transformation <- exp
  } else { 
    transformation <- NULL
  }
  
  
  if (plot == TRUE) {
    forest_plot <- forest(fitted_model,
                          xlab = effect_size_label,
                          addcred = TRUE,
                          atransf = transformation,
                          refline = reference_line)
  } else {
    forest_plot <- NULL
  }
  
  
  return(list(fitted_model = fitted_model, 
              meta_analysis_results = meta_analysis_results,
              meta_analysis_results_text = meta_analysis_results_text,
              heterogeneity_test_results_text = heterogeneity_test_results_text,
              forest_plot = forest_plot))
}


# robustness between original test and subset -----------
robust_test <- function(subset_test, original_test) {
  
  subset_p <- subset_test$estimate[subset_test$metric == "p"] %>%
    str_remove(., "=") %>%
    str_remove(., "<") %>%
    str_remove(., " ") %>%
    as.numeric()
  
  original_p <- original_test$estimate[original_test$metric == "p"] %>%
    str_remove(., "=") %>%
    str_remove(., "<") %>%
    str_remove(., " ") %>%
    as.numeric()
  
  return((subset_p >= 0.05 & original_p >= 0.05) | (subset_p < 0.05 & original_p < 0.05))
}


# moderator meta analysis workflow ------
moderator_meta_analysis_workflow <- function(data, 
                                             effect_size_label = "Cohen's d") {
  require(timesavers)
  require(tidyverse)
  require(metafor)
  
  # fit random effects model 
  fitted_model <- 
    rma(yi = yi, 
        sei = sei,
        data = data,
        mods = ~ exclude,
        slab = paste(Experiment, ifelse(exclude == TRUE, "excluded", "retained"), sep = " - "))

  # summarize moderator results
  mod_meta_analysis_results_text <- 
    paste0("QM(df = 1) = ", round(fitted_model$QM, 2), 
           ", p ", ifelse(fitted_model$QMp < 0.001, "< .001", as.character(paste("=", round(fitted_model$QMp, 3)))),
           ", difference in excluded subset ", effect_size_label, " = ", round(fitted_model$b[2], 2), 
           ", 95% CI [", round(fitted_model$ci.lb[2], 2), ", ", round(fitted_model$ci.ub[2], 2), "]")
  
  return(list(fitted_model = fitted_model, 
              mod_meta_analysis_results_text = mod_meta_analysis_results_text))
  
}

Experiments

Experiment 1

Demographics

data_exp_1 %>% 
  dplyr::summarize(n = n(),
                   age_mean = mean(Age),
                   age_sd = sd(Age)) %>%
  round_df(2) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

n	age_mean	age_sd
103	33.06	8.43

data_exp_1 %>% 
  dplyr::count(Gender) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

Gender	n
Female	57
Male	46

H1: IAT

results_exp_1_h1 <- data_exp_1 %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed an automatic relative preference for CS1 over CS2 on the IAT (M = 0.37, SD = 0.46), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = -0.23, SD = 0.45), t(98.12) = 6.63, p < .001, d = 1.31, 95% CI = [0.88, 1.74], BF10 = 5714391.64.

H2: Self-report ratings

results_exp_1_h2 <- data_exp_1 %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed a relative preference on the self-report ratings for CS1 over CS2 (M = 3.33, SD = 4.60), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = -4.15, SD = 4.48), t(98.32) = 8.33, p < .001, d = 1.65, 95% CI = [1.20, 2.10], BF10 = 15170767375.77.

H3: Behavioural intentions

# set reference level to "Morag"
data_exp_1$intention <- relevel(as.factor(data_exp_1$intention), ref = "Morag")

results_exp_1_h3 <- data_exp_1 %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

Data from the behavioural intentions question were entered into a multinomial logistic regression with CSI as the reference category. Only results from the CS1-CS2 comparison are relevant to the hypothesis and will be reported here. Results demonstrated that participants responses of CS1 relative to CS2 differed between the two conditions, congruent with training, OR = 13.66, 95% CI = [3.56, 52.44], p = .0001.

Experiment 2

Demographics

data_exp_2 %>% 
  dplyr::summarize(n = n(),
                   age_mean = mean(Age),
                   age_sd = sd(Age)) %>%
  round_df(2) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

n	age_mean	age_sd
103	32.5	8.87

data_exp_2 %>% 
  dplyr::count(Gender) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

Gender	n
Female	58
Male	45

H1: IAT

results_exp_2_h1 <- data_exp_2 %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS2 eventually shared a color with positive words and CS1 shared a color with negative words, participants showed an automatic relative preference for CS2 over CS1 on the IAT (M = -0.26, SD = 0.54). whereas when CS2 eventually shared a color with negative words and CS1 shared a color with positive words, they demonstrated a relative preference for CS1 over CS2 (M = -0.12, SD = 0.60), t(100.85) = -1.18, p = .24, d = -0.23, 95% CI = [-0.62, 0.16], BF10 = 0.38.

Note that this result is in the opposite direction to all other experiments.

H2: Self-report ratings

results_exp_2_h2 <- data_exp_2 %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

Regardless of whether CS1 eventually shared a color with positive or negative words, participants showed a relative preference on the self-report ratings for CS2 over CS1 (CS1-positive condition: M = -2.80, SD = 5.33; CS1-negative condition: M = -1.58, SD = 6.03), t(100.98) = -1.09, p = .28, d = -0.21, 95% CI = [-0.61, 0.18], BF10 = 0.35.

Note that this null result is different to all other experiments.

H3: Behavioural intentions

# set reference level to "Morag"
data_exp_2$intention <- relevel(as.factor(data_exp_2$intention), ref = "Morag")

results_exp_2_h3 <- data_exp_2 %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

Data from the behavioural intentions question were entered into a multinomial logistic regression with CSI as the reference category. Only results from the CS1-CS2 comparison are relevant to the hypothesis and will be reported here. Results demonstrated that participants responses of CS1 relative to CS2 differed between the two conditions, but in the opposite direction to that predicted, OR = 0.22, 95% CI = [0.07, 0.66], p = .0073.

Note that this result is in the opposite direction to all other experiments.

Distribution of IAT and ratings

Experiment 3 was informed by the unexpected results of experiment 2, which prompted us to examine the distribution of scores on the IAT and self reports which suggested bi- and tri-modality. This is depicted below.

data_exp_2 %>%
  select(Participant, source_valence_condition, IAT_Score, EC_Effect_CS1_CS2_Difference_Score) %>%
  rename(`IAT D score` = IAT_Score,
         `Ratings difference score` = EC_Effect_CS1_CS2_Difference_Score) %>%
  gather(dv_modality, dv_score, c(`IAT D score`, `Ratings difference score`)) %>%
  ggplot(aes(dv_score)) +
  geom_density(adjust = 0.5) +
  facet_wrap(~dv_modality+source_valence_condition, scales = "free") +
  theme_classic() +
  theme(panel.grid.major = element_blank(),
        panel.grid.minor = element_blank(),
        strip.background = element_blank(),
        panel.border = element_blank()) +
  xlab("") +
  ylab("Probability density")

Experiment 3

Demographics

data_exp_3 %>% 
  dplyr::summarize(n = n(),
                   age_mean = mean(Age),
                   age_sd = sd(Age)) %>%
  round_df(2) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

n	age_mean	age_sd
97	28.67	6.07

data_exp_3 %>% 
  dplyr::count(Gender) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

Gender	n
Female	58
Male	39

H1: IAT

results_exp_3_h1 <- data_exp_3 %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed an automatic relative preference for CS1 over CS2 on the IAT (M = 0.21, SD = 0.46), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = -0.15, SD = 0.59), t(93.42) = 3.29, p = .001, d = 0.66, 95% CI = [0.25, 1.08], BF10 = 20.10.

H2: Self-report ratings

results_exp_3_h2 <- data_exp_3 %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed a relative preference on the self-report ratings for CS1 over CS2 (M = 2.92, SD = 5.25), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = -2.85, SD = 5.02), t(92.94) = 5.52, p < .001, d = 1.13, 95% CI = [0.69, 1.56], BF10 = 45262.88.

H3: Behavioural intentions

# set reference level to "Morag"
data_exp_3$intention <- relevel(as.factor(data_exp_3$intention), ref = "Morag")

results_exp_3_h3 <- data_exp_3 %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

Data from the behavioural intentions question were entered into a multinomial logistic regression with CSI as the reference category. Only results from the CS1-CS2 comparison are relevant to the hypothesis and will be reported here. Results demonstrated that participants responses of CS1 relative to CS2 differed between the two conditions, congruent with training, OR = 6.94, 95% CI = [2.03, 23.77], p = .002.

Experiment 4

Demographics

data_exp_4 %>% 
  dplyr::summarize(n = n(),
                   age_mean = mean(Age),
                   age_sd = sd(Age)) %>%
  round_df(2) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

n	age_mean	age_sd
187	30.3	6.2

data_exp_4 %>% 
  dplyr::count(Gender) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

Gender	n
Female	94
Male	93

H1: IAT

results_exp_4_h1 <- data_exp_4 %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed an automatic relative preference for CS1 over CS2 on the IAT (M = 0.16, SD = 0.48), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = -0.18, SD = 0.46), t(184.00) = 5.02, p < .001, d = 0.73, 95% CI = [0.44, 1.03], BF10 = 11902.22.

H2: Self-report ratings

results_exp_4_h2 <- data_exp_4 %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed a relative preference on the self-report ratings for CS1 over CS2 (M = 3.57, SD = 4.99), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = -2.18, SD = 4.21), t(179.27) = 8.51, p < .001, d = 1.25, 95% CI = [0.93, 1.56], BF10 = 922634733581.12.

H3: Behavioural intentions

# set reference level to "Morag"
data_exp_4$intention <- relevel(as.factor(data_exp_4$intention), ref = "Morag")

results_exp_4_h3 <- data_exp_4 %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

Data from the behavioural intentions question were entered into a multinomial logistic regression with CSI as the reference category. Only results from the CS1-CS2 comparison are relevant to the hypothesis and will be reported here. Results demonstrated that participants responses of CS1 relative to CS2 differed between the two conditions, congruent with training, OR = 7.63, 95% CI = [3.11, 18.76], p < .0001.

Experiment 5

Demographics

data_exp_5 %>% 
  dplyr::summarize(n = n(),
                   age_mean = mean(Age),
                   age_sd = sd(Age)) %>%
  round_df(2) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

n	age_mean	age_sd
179	30.92	5.85

data_exp_5 %>% 
  dplyr::count(Gender) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

Gender	n
Female	95
Male	84

H1: IAT

results_exp_5_h1 <- data_exp_5 %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed an automatic relative preference for CS1 over CS2 on the IAT (M = 0.14, SD = 0.46), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = -0.12, SD = 0.46), t(168.75) = 3.79, p < .001, d = 0.57, 95% CI = [0.27, 0.87], BF10 = 109.05.

H2: Self-report ratings

results_exp_5_h2 <- data_exp_5 %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed a relative preference on the self-report ratings for CS1 over CS2 (M = 2.34, SD = 4.12), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = -2.38, SD = 4.09), t(169.77) = 7.66, p < .001, d = 1.15, 95% CI = [0.83, 1.47], BF10 = 5415593221.07.

H3: Behavioural intentions

# set reference level to "Morag"
data_exp_5$intention <- relevel(as.factor(data_exp_5$intention), ref = "Morag")

results_exp_5_h3 <- data_exp_5 %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

Data from the behavioural intentions question were entered into a multinomial logistic regression with CSI as the reference category. Only results from the CS1-CS2 comparison are relevant to the hypothesis and will be reported here. Results demonstrated that participants responses of CS1 relative to CS2 differed between the two conditions, congruent with training, OR = 5.00, 95% CI = [1.91, 13.06], p = .001.

Experiment 6

Demographics

data_exp_6 %>% 
  dplyr::summarize(n = n(),
                   age_mean = mean(Age),
                   age_sd = sd(Age)) %>%
  round_df(2) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

n	age_mean	age_sd
231	27.34	7.33

data_exp_6 %>% 
  dplyr::count(Gender) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

Gender	n
female	103
male	125
other	3

H1: IAT

results_exp_6_h1 <- data_exp_6 %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed an automatic relative preference for CS1 over CS2 on the IAT (M = 0.31, SD = 0.51), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = -0.18, SD = 0.52), t(227.66) = 7.25, p < .001, d = 0.96, 95% CI = [0.68, 1.23], BF10 = 1151492207.55.

H2: Self-report ratings

results_exp_6_h2 <- data_exp_6 %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed a relative preference on the self-report ratings for CS1 over CS2 (M = 3.61, SD = 4.72), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = -3.40, SD = 4.09), t(227.44) = 12.08, p < .001, d = 1.58, 95% CI = [1.29, 1.88], BF10 = 72613291696428081479680.00.

H3: Behavioural intentions

# set reference level to "Morag"
data_exp_6$intention <- relevel(as.factor(data_exp_6$intention), ref = "Morag")

results_exp_6_h3 <- data_exp_6 %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

Data from the behavioural intentions question were entered into a multinomial logistic regression with CSI as the reference category. Only results from the CS1-CS2 comparison are relevant to the hypothesis and will be reported here. Results demonstrated that participants responses of CS1 relative to CS2 differed between the two conditions, congruent with training, OR = 14.85, 95% CI = [6.98, 31.63], p < .0001.

Experiment 7

Demographics

data_exp_7 %>% 
  dplyr::summarize(n = n(),
                   age_mean = mean(Age, na.rm = TRUE),
                   age_sd = sd(Age, na.rm = TRUE)) %>%
  round_df(2) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

n	age_mean	age_sd
493	28.28	7.43

data_exp_7 %>% 
  dplyr::count(Gender) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

Gender	n
female	242
male	246
NA	5

BF analyses of Evaluative Priming(preregistered)

N = 300

These analyses code each trial as being congruent or incongruent based on the valence of the prime and the valence of the source stimulus valence. Mean reaction time was calculated for each participant’s congruent and incongruent trials. These means for each trial type are then compared between participants. This represents the classical test of the EP effect.

data_300 <- data_exp_7 %>%
  slice(1:300) %>%
  select(ep_mean_rt_consistent, ep_mean_rt_inconsistent) %>%
  gather()

ttestBF(x = data_300$value[data_300$key == "ep_mean_rt_inconsistent"],
        y = data_300$value[data_300$key == "ep_mean_rt_consistent"],
        paired = TRUE)

## Bayes factor analysis
## --------------
## [1] Alt., r=0.707 : 1.41985 ±0%
## 
## Against denominator:
##   Null, mu = 0 
## ---
## Bayes factor type: BFoneSample, JZS

sample_300 <- 
  ttestBF(x = data_300$value[data_300$key == "ep_mean_rt_inconsistent"],
          y = data_300$value[data_300$key == "ep_mean_rt_consistent"],
          paired = TRUE,
          posterior = TRUE, 
          iterations = 5000)

data.frame(sample_300) %>%
  dplyr::summarize(mu_median = quantile(mu, prob = .500),
                   mu_lower = quantile(mu, prob = .025),
                   mu_upper = quantile(mu, prob = .975),
                   delta_median = quantile(delta, prob = .500),
                   delta_lower = quantile(delta, prob = .025),
                   delta_upper = quantile(delta, prob = .975)) %>%
  round(2) %>%
  gather(parameter, estimate)

##      parameter estimate
## 1    mu_median     4.21
## 2     mu_lower     0.95
## 3     mu_upper     7.49
## 4 delta_median     0.14
## 5  delta_lower     0.03
## 6  delta_upper     0.26

N = 400

data_400 <- data_exp_7 %>%
  slice(1:400) %>%
  select(ep_mean_rt_consistent, ep_mean_rt_inconsistent) %>%
  gather()

ttestBF(x = data_400$value[data_400$key == "ep_mean_rt_inconsistent"],
        y = data_400$value[data_400$key == "ep_mean_rt_consistent"],
        paired = TRUE)

## Bayes factor analysis
## --------------
## [1] Alt., r=0.707 : 0.7645194 ±0.01%
## 
## Against denominator:
##   Null, mu = 0 
## ---
## Bayes factor type: BFoneSample, JZS

sample_400 <- 
  ttestBF(x = data_400$value[data_400$key == "ep_mean_rt_inconsistent"],
          y = data_400$value[data_400$key == "ep_mean_rt_consistent"],
          paired = TRUE,
          posterior = TRUE, 
          iterations = 5000)

data.frame(sample_400) %>%
  dplyr::summarize(mu_median = quantile(mu, prob = .500),
                   mu_lower = quantile(mu, prob = .025),
                   mu_upper = quantile(mu, prob = .975),
                   delta_median = quantile(delta, prob = .500),
                   delta_lower = quantile(delta, prob = .025),
                   delta_upper = quantile(delta, prob = .975)) %>%
  round(2) %>%
  gather(parameter, estimate)

##      parameter estimate
## 1    mu_median     3.39
## 2     mu_lower     0.51
## 3     mu_upper     6.29
## 4 delta_median     0.11
## 5  delta_lower     0.02
## 6  delta_upper     0.21

N = 500

data_500 <- data_exp_7 %>%
  select(ep_mean_rt_consistent, ep_mean_rt_inconsistent) %>%
  gather()

ttestBF(x = data_500$value[data_500$key == "ep_mean_rt_inconsistent"],
        y = data_500$value[data_500$key == "ep_mean_rt_consistent"],
        paired = TRUE)

## Bayes factor analysis
## --------------
## [1] Alt., r=0.707 : 3.113847 ±0%
## 
## Against denominator:
##   Null, mu = 0 
## ---
## Bayes factor type: BFoneSample, JZS

sample_500 <- 
  ttestBF(x = data_500$value[data_500$key == "ep_mean_rt_inconsistent"],
          y = data_500$value[data_500$key == "ep_mean_rt_consistent"],
          paired = TRUE,
          posterior = TRUE, 
          iterations = 5000)

data.frame(sample_500) %>%
  dplyr::summarize(mu_median = quantile(mu, prob = .500),
                   mu_lower = quantile(mu, prob = .025),
                   mu_upper = quantile(mu, prob = .975),
                   delta_median = quantile(delta, prob = .500),
                   delta_lower = quantile(delta, prob = .025),
                   delta_upper = quantile(delta, prob = .975)) %>%
  round(2) %>%
  gather(parameter, estimate)

##      parameter estimate
## 1    mu_median     3.88
## 2     mu_lower     1.28
## 3     mu_upper     6.58
## 4 delta_median     0.13
## 5  delta_lower     0.04
## 6  delta_upper     0.22

Combined frequentist and BF analyses (not preregistered, included for consistency across studies)

All use the final analytic sample.

H1: Evaluative Priming

Unlike the above tests which were used for sample size determination via optional stopping, these analyses are consistent with the anlayses of the IATs and self-reports, and behavioural intentions in the other studies. Specficially, 1) a difference score was first calculated for each participant between their mean RT on congruent trials vs incongruent trials, 2) this difference score was then inverted (ie multiplied by -1) for one stimulus valence condition. These difference scores were then compared between conditions. As such, the current analyses test whether there is a difference in EP effect between source valence conditions, whereas the above one provided a more classic test of the EP effect (congruent vs incongruent trials) while remaining agnostic to the source valence condition (i.e., integrating this condition into the scoring of [in]congruence rather than the analysis). As such, the BF provided by the two analyses may differ.

results_exp_7_h1 <- data_exp_7 %>%
  mutate(ep_mean_rt_difference = ifelse(source_valence_condition == "CS2_Positive_CS1_Negative", ep_mean_rt_difference*-1, 
                                        ifelse(source_valence_condition == "CS1_Positive_CS2_Negative", ep_mean_rt_difference, NA))) %>%
  select(ep_mean_rt_difference, source_valence_condition) %>%
  na.omit() %>%
  mutate(DV = ep_mean_rt_difference,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed an automatic relative preference for CS1 over CS2 on the IAT (M = 3.59, SD = 28.94), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = -3.81, SD = 30.78), t(485.60) = 2.74, p = .006, d = 0.25, 95% CI = [0.07, 0.43], BF10 = 3.72.

H2: Self-report ratings

results_exp_7_h2 <- data_exp_7 %>%
  select(EC_Effect_CS1_CS2_Difference_Score, source_valence_condition) %>%
  na.omit() %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed a relative preference on the self-report ratings for CS1 over CS2 (M = 2.17, SD = 4.35), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = -2.66, SD = 4.02), t(479.98) = 12.72, p < .001, d = 1.15, 95% CI = [0.96, 1.35], BF10 = 80819093185460097270287433728.00.

H3: Behavioural intentions

# set reference level to "Morag"
data_exp_7$intention <- relevel(as.factor(data_exp_7$intention), ref = "Morag")

results_exp_7_h3 <- data_exp_7 %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

Data from the behavioural intentions question were entered into a multinomial logistic regression with CSI as the reference category. Only results from the CS1-CS2 comparison are relevant to the hypothesis and will be reported here. Results demonstrated that participants responses of CS1 relative to CS2 differed between the two conditions, congruent with training, OR = 4.33, 95% CI = [2.77, 6.78], p < .0001.

Experiment 8

Demographics

data_exp_8 %>% 
  dplyr::summarize(n = n(),
                   age_mean = mean(Age),
                   age_sd = sd(Age)) %>%
  round_df(2) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

n	age_mean	age_sd
222	28.84	7.54

data_exp_8 %>% 
  dplyr::count(Gender) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

Gender	n
female	127
male	92
other	3

Comparing conditions

Pairwise follow up t-tests using Bonferroni-Holm corrections for multiple testing.

Results are not parsed into text automatically given the analysis workflow differs from other experiments, and this would take more coding effort than its worth.

H1: IAT

# anova
fit_exp_8_h1 <- aov(IAT_Score ~ Bag_Matching, 
                    contrasts = "contr.sum",
                    data = data_exp_8)

Anova(fit_exp_8_h1, type = "III")

## Anova Table (Type III tests)
## 
## Response: IAT_Score
##              Sum Sq  Df F value           Pr(>F)    
## (Intercept)   3.432   1  19.854 0.00001333305091 ***
## Bag_Matching  9.770   2  28.261 0.00000000001206 ***
## Residuals    37.854 219                             
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

# effect size
etaSquared(fit_exp_8_h1, type = 2, anova = FALSE )

##                 eta.sq eta.sq.part
## Bag_Matching 0.2051465   0.2051465

# follow up t tests
pairwise.t.test(data_exp_8$IAT_Score, 
                data_exp_8$Bag_Matching, 
                p.adj = "holm")

## 
##  Pairwise comparisons using t tests with pooled SD 
## 
## data:  data_exp_8$IAT_Score and data_exp_8$Bag_Matching 
## 
##                                         No_Shared_Feature_Condition_2
## Shared_Feature_Bag_Location_Condition_1 0.000000000041               
## Shared_Feature_Bag_Random_Condition_3   0.000000515586               
##                                         Shared_Feature_Bag_Location_Condition_1
## Shared_Feature_Bag_Location_Condition_1 -                                      
## Shared_Feature_Bag_Random_Condition_3   0.1                                    
## 
## P value adjustment method: holm

# means by condition
data_exp_8 %>%
  group_by(Bag_Matching) %>%
  dplyr::summarize(mean = mean(IAT_Score),
                   ci_lwr = mean - plotrix::std.error(IAT_Score)*1.96,
                   ci_upr = mean + plotrix::std.error(IAT_Score)*1.96) %>%
  round_df(2)

## # A tibble: 3 x 4
##   Bag_Matching                             mean ci_lwr ci_upr
##   <chr>                                   <dbl>  <dbl>  <dbl>
## 1 No_Shared_Feature_Condition_2           -0.2  -0.290  -0.12
## 2 Shared_Feature_Bag_Location_Condition_1  0.28  0.18    0.38
## 3 Shared_Feature_Bag_Random_Condition_3    0.16  0.06    0.26

H2: Self-report ratings

# anova
fit_exp_8_h2 <- aov(EC_Effect_CS1_CS2_Difference_Score ~ Bag_Matching, 
                    contrasts = "contr.sum",
                    data = data_exp_8)

Anova(fit_exp_8_h2, type = "III")

## Anova Table (Type III tests)
## 
## Response: EC_Effect_CS1_CS2_Difference_Score
##               Sum Sq  Df F value          Pr(>F)    
## (Intercept)   109.14   1  8.9765         0.00305 ** 
## Bag_Matching  649.36   2 26.7035 0.0000000000419 ***
## Residuals    2662.77 219                            
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

# effect size
etaSquared(fit_exp_8_h2, type = 2, anova = FALSE )

##                eta.sq eta.sq.part
## Bag_Matching 0.196056    0.196056

# follow up t tests
pairwise.t.test(data_exp_8$EC_Effect_CS1_CS2_Difference_Score, 
                data_exp_8$Bag_Matching, 
                p.adj = "holm")

## 
##  Pairwise comparisons using t tests with pooled SD 
## 
## data:  data_exp_8$EC_Effect_CS1_CS2_Difference_Score and data_exp_8$Bag_Matching 
## 
##                                         No_Shared_Feature_Condition_2
## Shared_Feature_Bag_Location_Condition_1 0.00000000062                
## Shared_Feature_Bag_Random_Condition_3   0.00000008789                
##                                         Shared_Feature_Bag_Location_Condition_1
## Shared_Feature_Bag_Location_Condition_1 -                                      
## Shared_Feature_Bag_Random_Condition_3   0.39                                   
## 
## P value adjustment method: holm

# means by condition
data_exp_8 %>%
  group_by(Bag_Matching) %>%
  dplyr::summarize(mean = mean(EC_Effect_CS1_CS2_Difference_Score),
                   ci_lwr = mean - plotrix::std.error(EC_Effect_CS1_CS2_Difference_Score)*1.96,
                   ci_upr = mean + plotrix::std.error(EC_Effect_CS1_CS2_Difference_Score)*1.96) %>%
  round_df(2)

## # A tibble: 3 x 4
##   Bag_Matching                             mean ci_lwr ci_upr
##   <chr>                                   <dbl>  <dbl>  <dbl>
## 1 No_Shared_Feature_Condition_2           -1.14  -1.78  -0.5 
## 2 Shared_Feature_Bag_Location_Condition_1  2.61   1.73   3.49
## 3 Shared_Feature_Bag_Random_Condition_3    2.1    1.21   3

Individual conditions for meta analysis

Standard non-random instructions condition

H1: IAT

results_exp_8nri_h1 <- data_exp_8nri %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed an automatic relative preference for CS1 over CS2 on the IAT (M = 0.44, SD = 0.32), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = 0.10, SD = 0.46), t(58.42) = 3.50, p < .001, d = 0.84, 95% CI = [0.35, 1.34], BF10 = 42.20.

H2: Self-report ratings

results_exp_8nri_h2 <- data_exp_8nri %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed a relative preference on the self-report ratings for CS1 over CS2 (M = 3.57, SD = 2.82), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = 1.57, SD = 4.41), t(55.35) = 2.26, p = .03, d = 0.55, 95% CI = [0.06, 1.03], BF10 = 2.25.

Random instructions condition for meta analysis

H1: IAT

results_exp_8ri_h1 <- data_exp_8ri %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed an automatic relative preference for CS1 over CS2 on the IAT (M = 0.45, SD = 0.40), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = 0.01, SD = 0.35), t(40.42) = 4.53, p < .001, d = 1.21, 95% CI = [0.66, 1.77], BF10 = 1255.98.

H2: Self-report ratings

results_exp_8ri_h2 <- data_exp_8ri %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed a relative preference on the self-report ratings for CS1 over CS2 (M = 3.53, SD = 3.90), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = 1.35, SD = 3.46), t(40.33) = 2.26, p = .03, d = 0.60, 95% CI = [0.08, 1.13], BF10 = 2.52.

Experiment 9

Demographics

data_exp_9 %>% 
  dplyr::summarize(n = n(),
                   age_mean = mean(Age),
                   age_sd = sd(Age)) %>%
  round_df(2) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

n	age_mean	age_sd
89	31.53	7.35

data_exp_9 %>% 
  dplyr::count(Gender) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

Gender	n
female	47
male	42

H1: IAT

t.test(data_exp_9$IAT_Score, alternative = "two.sided")

## 
##  One Sample t-test
## 
## data:  data_exp_9$IAT_Score
## t = -3.8952, df = 88, p-value = 0.0001908
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.24189282 -0.07845558
## sample estimates:
##  mean of x 
## -0.1601742

data_exp_9 %>%
  dplyr::summarize(d = round(mean(IAT_Score)/sd(IAT_Score), 2))

## # A tibble: 1 x 1
##       d
##   <dbl>
## 1 -0.41

# ggplot(data_exp_9, aes(IAT_Score)) +
#   geom_density()

H2: Self-report ratings

t.test(data_exp_9$EC_Effect_CS1_CS2_Difference_Score, alternative = "two.sided")

## 
##  One Sample t-test
## 
## data:  data_exp_9$EC_Effect_CS1_CS2_Difference_Score
## t = -5.7116, df = 88, p-value = 0.0000001495
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -2.101422 -1.016555
## sample estimates:
## mean of x 
## -1.558989

data_exp_9 %>%
  dplyr::summarize(d = round(mean(EC_Effect_CS1_CS2_Difference_Score)/sd(EC_Effect_CS1_CS2_Difference_Score), 2))

## # A tibble: 1 x 1
##       d
##   <dbl>
## 1 -0.61

# ggplot(data_exp_9, aes(EC_Effect_CS1_CS2_Difference_Score)) +
#   geom_density()

Meta analyses

Meta analyses of experiments 1-8 (nb not 9, which contained no manipulations but acted as an extension of experiment 8).

Random effects meta analyses were fitted using the metafor R package (Viechtbauer, 2010). The maximum liklihood estimator was used.

A separate meta analysis was fitted for each outcome variable (IAT, self reported ratings, behavioural intentions). Although multivariate meta analysis was a possible alternative (and is conducted in a separate markdown script), previous work has traditionally assessed these dependant variables separately, so we will too.

Sample sizes

Note that not all variables were collected in all subsets, so sample sizes for subsets do not equal the total sample size * the proportion meeting a given criteria.

data_combined <- 
  bind_rows(mutate(data_exp_1, experiment = 1),
            mutate(data_exp_2, experiment = 2),
            mutate(data_exp_3, experiment = 3),
            mutate(data_exp_4, experiment = 4),
            mutate(data_exp_5, experiment = 5),
            mutate(data_exp_6, experiment = 6),
            mutate(data_exp_7, experiment = 7),
            mutate(data_exp_8nri, experiment = 8),
            mutate(data_exp_8ri, experiment = 8)) %>%
  select(experiment, US_contingency_memory, color_valence_contingency_memory, 
         demand_IAT, demand_self_reports, hypothesis_awareness, reactance_IAT, reactance_self_reports) %>%
  # fudge to stop "unknown variable" warning from appearing
  mutate(DV = NA)

data_combined %>% 
  dplyr::summarize(full_sample_n = n()) %>%
  round_df(2) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

full_sample_n
1531

# percents by subset
temp_1 <- data_combined %>%
  count(US_contingency_memory) %>%
  spread(US_contingency_memory, n) %>%
  mutate(US_contingency_aware = `TRUE` / (`TRUE` + `FALSE`)) %>%
  select(US_contingency_aware)

temp_2 <- data_combined %>%
  count(color_valence_contingency_memory) %>%
  spread(color_valence_contingency_memory, n) %>%
  mutate(color_valence_contingency_aware = `TRUE` / (`TRUE` + `FALSE`)) %>%
  select(color_valence_contingency_aware)

temp_3 <- data_combined %>%
  rowwise() %>%
  mutate(demand = as.logical(max(c(demand_IAT, demand_self_reports)))) %>%
  ungroup() %>%
  count(demand) %>%
  spread(demand, n) %>%
  mutate(NOT_demand_compliant = `FALSE` / (`TRUE` + `FALSE`)) %>%
  select(NOT_demand_compliant)

temp_4a <- data_combined %>%
  count(reactance_IAT) %>%
  spread(reactance_IAT, n) %>%
  mutate(NOT_reactant_IAT = `FALSE` / (`TRUE` + `FALSE`)) %>%
  select(NOT_reactant_IAT)

temp_4b <- data_combined %>%
  count(reactance_self_reports) %>%
  spread(reactance_self_reports, n) %>%
  mutate(NOT_reactant_self_reports = `FALSE` / (`TRUE` + `FALSE`)) %>%
  select(NOT_reactant_self_reports)

temp_5 <- data_combined %>%
  count(hypothesis_awareness) %>%
  spread(hypothesis_awareness, n) %>%
  mutate(NOT_hypothesis_awareness = `FALSE` / (`TRUE` + `FALSE`)) %>%
  select(NOT_hypothesis_awareness)

bind_cols(temp_1,
          temp_2,
          temp_3,
          temp_4a,
          temp_4b,
          temp_5) %>%
  gather(subset, proportion) %>%
  round_df(3) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

subset	proportion
US_contingency_aware	0.775
color_valence_contingency_aware	0.596
NOT_demand_compliant	0.809
NOT_reactant_IAT	0.897
NOT_reactant_self_reports	0.858
NOT_hypothesis_awareness	0.267

# percents by subset
temp_1_by_exp <- data_combined %>%
  count(experiment, US_contingency_memory) %>%
  spread(US_contingency_memory, n) %>%
  mutate(US_contingency_aware = `TRUE` / (`TRUE` + `FALSE`)) %>%
  select(experiment, US_contingency_aware)

temp_2_by_exp <- data_combined %>%
  count(experiment, color_valence_contingency_memory) %>%
  spread(color_valence_contingency_memory, n) %>%
  mutate(color_valence_contingency_aware = `TRUE` / (`TRUE` + `FALSE`)) %>%
  select(experiment, color_valence_contingency_aware)

temp_3_by_exp <- data_combined %>%
  rowwise() %>%
  mutate(demand = as.logical(max(c(demand_IAT, demand_self_reports)))) %>%
  ungroup() %>%
  count(experiment, demand) %>%
  spread(demand, n) %>%
  mutate(NOT_demand_compliant = `FALSE` / (`TRUE` + `FALSE`)) %>%
  select(experiment, NOT_demand_compliant)

temp_4a_by_exp <- data_combined %>%
  count(experiment, reactance_IAT) %>%
  spread(reactance_IAT, n) %>%
  mutate(NOT_reactant_IAT = `FALSE` / (`TRUE` + `FALSE`)) %>%
  select(experiment, NOT_reactant_IAT)

temp_4b_by_exp <- data_combined %>%
  count(experiment, reactance_self_reports) %>%
  spread(reactance_self_reports, n) %>%
  mutate(NOT_reactant_self_reports = `FALSE` / (`TRUE` + `FALSE`)) %>%
  select(experiment, NOT_reactant_self_reports)

temp_5_by_exp <- data_combined %>%
  count(experiment, hypothesis_awareness) %>%
  spread(hypothesis_awareness, n) %>%
  mutate(NOT_hypothesis_awareness = `FALSE` / (`TRUE` + `FALSE`)) %>%
  select(experiment, NOT_hypothesis_awareness)

temp_1_by_exp %>%
  left_join(temp_2_by_exp, by = "experiment") %>%
  left_join(temp_3_by_exp, by = "experiment") %>%
  left_join(temp_4a_by_exp, by = "experiment") %>%
  left_join(temp_4b_by_exp, by = "experiment") %>%
  left_join(temp_5_by_exp, by = "experiment") %>%
  round_df(3) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

experiment	US_contingency_aware	color_valence_contingency_aware	NOT_demand_compliant	NOT_reactant_IAT	NOT_reactant_self_reports	NOT_hypothesis_awareness
1	0.650	0.718	0.728	0.806	0.835	0.282
2	0.650	0.621	0.816	0.903	0.845	0.184
3	0.794	0.794	0.701	0.887	0.856	0.082
4	0.781	0.620	0.802	0.904	0.850	0.326
5	0.749	0.453	0.698	0.916	0.849	0.140
6	NA	NA	0.913	0.926	0.848	0.329
7	0.781	0.568	0.824	0.895	0.900	0.348
8	0.949	NA	0.862	0.891	0.775	0.145

Meta analysed effect size

Notes on meta analysed effect size:

Credibility intervals refer to the interval of effect sizes likely to be observed in a future study on the basis of the width of estimation of the true effect size combined with variation in this true effect size (e.g., due to heterogeneity).

Notes on heterogeneity metrics:

• Q and its p value: A measure of weighted squared deviations. Depends on number of studies.
• tau^2: Population between-studies variance, ie estimated amount of total true heterogeneity in the effect. Is on the same scale as the meta analysed effect size, and so cannot be compared between meta analysis using different scales (e.g., Cohen’s d vs. Odds Ratios).
• I^2: Percentage of variation in the observed effects that is due to true heterogeneity (as quantified by tau^2) as opposed to sampling variation. Metric is therefore a description of data in sample, not an underlying quality associated with the true effect. Can be thought of as analogous to the reliability of a scale, which also represents the percentage of true variance as portion of total variance. Range 0-100, lower values preferable. E.g., 0 = no variability in observed effects to be explained as a systematic influence (e.g., some due to some moderator) as its all just sampling variation. Does not depend on the effect size scale or the number of studies.
• H^2: Total variability / sampling variability, i.e., (true variability / sampling variability)/sampling variability. Lower values preferable, i.e., refer to less true variation to be explained as systematic (e.g., due to some moderator).

H1: IAT

results_for_meta_analysis_h1 <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", "Experiment 6",
                            "Experiment 8 (standard)", 
                            "Experiment 8 (random)"), 
             cohens_d = c(results_exp_1_h1$cohens_d$estimate[[1]], 
                          results_exp_2_h1$cohens_d$estimate[[1]], 
                          results_exp_3_h1$cohens_d$estimate[[1]], 
                          results_exp_4_h1$cohens_d$estimate[[1]],
                          results_exp_5_h1$cohens_d$estimate[[1]],
                          results_exp_6_h1$cohens_d$estimate[[1]],
                          results_exp_8nri_h1$cohens_d$estimate[[1]], 
                          results_exp_8ri_h1$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h1$cohens_d$conf.int[["lower"]],
                        results_exp_2_h1$cohens_d$conf.int[["lower"]],
                        results_exp_3_h1$cohens_d$conf.int[["lower"]],
                        results_exp_4_h1$cohens_d$conf.int[["lower"]],
                        results_exp_5_h1$cohens_d$conf.int[["lower"]],
                        results_exp_6_h1$cohens_d$conf.int[["lower"]],
                        results_exp_8nri_h1$cohens_d$conf.int[["lower"]],
                        results_exp_8ri_h1$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h1$cohens_d$conf.int[["upper"]],
                        results_exp_2_h1$cohens_d$conf.int[["upper"]],
                        results_exp_3_h1$cohens_d$conf.int[["upper"]],
                        results_exp_4_h1$cohens_d$conf.int[["upper"]],
                        results_exp_5_h1$cohens_d$conf.int[["upper"]],
                        results_exp_6_h1$cohens_d$conf.int[["upper"]],
                        results_exp_8nri_h1$cohens_d$conf.int[["upper"]],
                        results_exp_8ri_h1$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2))

results_for_meta_analysis_h1 %>%
  select(Experiment, yi, sei) %>%
  write_csv("../meta analysis data/results_for_meta_analysis_iat.csv")

meta_results_h1 <- meta_analysis_workflow(data              = results_for_meta_analysis_h1,
                                          effect_size_label = "Cohen's d",
                                          exp_estimates     = FALSE,
                                          reference_line    = 0)

• Results demonstrated a significant effect of medium size (Cohen, 1988): k = 8, Cohen’s d = 0.75, 95% CI = [0.42, 1.07], 95% CR = [-0.14, 1.63], p < .0001
• Results were found to contain a high degree of heterogeneity: Q(df = 7) = 36.81, p < .001, tau^2 = 0.18, I^2 = 83.28%, H^2 = 5.98

Outlier tests

Undue influence of one or more studies on either the meta analysed effect size or the heterogeneity among studies was then assessed.

Note on influence metrics from the metafor package documentation:

• The DFFITS value indicates how many standard deviations the predicted (average) effect for the ith study changes after excluding the ith study from the model fitting.
• The leave-one-out amount of (residual) heterogeneity is the estimated value of τ based on the dataset with the ith study removed. Note that this is always equal to 0 for fixed-effects models.
• Similarly, the leave-one-out test statistic for the test of (residual) heterogeneity is the value of the test statistic of the test for (residual) heterogeneity calculated based on the dataset with the ith study removed.

Outliers were assessed on the baiss of undue influence, on the predicted average effect size/coefficients (DFFITS), and the large decrease in heterogeneity on the basis of its removal (tau2.del, QE.del). Note that outliers were also selected based on consistency across all three DVs (see other analyses below).

influence_meta_results_h1 <- influence(meta_results_h1$fitted_model)

print(influence_meta_results_h1, digits = 2) 

## 
##                         rstudent dffits cook.d cov.r tau2.del QE.del  hat 
## Experiment 1                1.35   0.51   0.23  1.01     0.15  29.06 0.12 
## Experiment 2               -3.84  -1.25   0.51  0.36     0.03  11.24 0.13 
## Experiment 3               -0.18  -0.08   0.01  1.33     0.21  36.70 0.12 
## Experiment 4               -0.03  -0.02   0.00  1.37     0.22  36.81 0.14 
## Experiment 5               -0.39  -0.16   0.03  1.34     0.21  35.54 0.14 
## Experiment 6                0.47   0.18   0.04  1.32     0.21  33.41 0.14 
## Experiment 8 (standard)     0.20   0.06   0.00  1.30     0.21  36.58 0.11 
## Experiment 8 (random)       0.97   0.33   0.11  1.13     0.18  33.74 0.11 
##                         weight  dfbs inf 
## Experiment 1             12.10  0.51     
## Experiment 2             12.56 -1.27   * 
## Experiment 3             12.30 -0.08     
## Experiment 4             13.63 -0.02     
## Experiment 5             13.58 -0.16     
## Experiment 6             13.88  0.18     
## Experiment 8 (standard)  11.33  0.06     
## Experiment 8 (random)    10.61  0.33

Experiment 2 was assessed as being an outlier.

Sensitivity analysis

A sensitivity analysis therefore excluded Experiment 2 and re meta analysed.

results_for_meta_analysis_h1_sensitivity <- results_for_meta_analysis_h1 %>%
  filter(Experiment != "Experiment 2")

meta_results_h1_sensitivity <- 
  meta_analysis_workflow(data              = results_for_meta_analysis_h1_sensitivity,
                         effect_size_label = "Cohen's d",
                         exp_estimates     = FALSE,
                         reference_line    = 0)

• After excluding one experiment as an outlier, the meta analysed effect size was found to be of large size: k = 7, Cohen’s d = 0.86, 95% CI = [0.67, 1.06], 95% CR = [0.47, 1.26], p < .0001, and heterogeneity was found to lower, Q(df = 6) = 11.24, p < .001, tau^2 = 0.03, I^2 = 47.02%, H^2 = 1.89.

H2: Self-report ratings

results_for_meta_analysis_h2 <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", "Experiment 6",
                            "Experiment 7",
                            "Experiment 8 (standard)", 
                            "Experiment 8 (random)"),
             cohens_d = c(results_exp_1_h2$cohens_d$estimate[[1]], 
                          results_exp_2_h2$cohens_d$estimate[[1]], 
                          results_exp_3_h2$cohens_d$estimate[[1]], 
                          results_exp_4_h2$cohens_d$estimate[[1]],
                          results_exp_5_h2$cohens_d$estimate[[1]],
                          results_exp_6_h2$cohens_d$estimate[[1]],
                          results_exp_7_h2$cohens_d$estimate[[1]],
                          results_exp_8nri_h2$cohens_d$estimate[[1]],
                          results_exp_8ri_h2$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h2$cohens_d$conf.int[["lower"]],
                        results_exp_2_h2$cohens_d$conf.int[["lower"]],
                        results_exp_3_h2$cohens_d$conf.int[["lower"]],
                        results_exp_4_h2$cohens_d$conf.int[["lower"]],
                        results_exp_5_h2$cohens_d$conf.int[["lower"]],
                        results_exp_6_h2$cohens_d$conf.int[["lower"]],
                        results_exp_7_h2$cohens_d$conf.int[["lower"]],
                        results_exp_8nri_h2$cohens_d$conf.int[["lower"]],
                        results_exp_8ri_h2$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h2$cohens_d$conf.int[["upper"]],
                        results_exp_2_h2$cohens_d$conf.int[["upper"]],
                        results_exp_3_h2$cohens_d$conf.int[["upper"]],
                        results_exp_4_h2$cohens_d$conf.int[["upper"]],
                        results_exp_5_h2$cohens_d$conf.int[["upper"]],
                        results_exp_6_h2$cohens_d$conf.int[["upper"]],
                        results_exp_7_h2$cohens_d$conf.int[["upper"]],
                        results_exp_8nri_h2$cohens_d$conf.int[["upper"]],
                        results_exp_8ri_h2$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2))

results_for_meta_analysis_h2 %>%
  select(Experiment, yi, sei) %>%
  write_csv("../meta analysis data/results_for_meta_analysis_self_report.csv")

meta_results_h2 <- meta_analysis_workflow(data              = results_for_meta_analysis_h2,
                                          effect_size_label = "Cohen's d",
                                          exp_estimates     = FALSE,
                                          reference_line    = 0)

• Results demonstrated a significant effect of large size: k = 9, Cohen’s d = 0.99, 95% CI = [0.62, 1.37], 95% CR = [-0.14, 2.12], p < .0001
• Results were found to contain a high degree of heterogeneity: Q(df = 8) = 68.37, p < .001, tau^2 = 0.29, I^2 = 90.65%, H^2 = 10.7

Outlier tests

influence_meta_results_h2 <- influence(meta_results_h2$fitted_model)

print(influence_meta_results_h2, digits = 2) 

## 
##                         rstudent dffits cook.d cov.r tau2.del QE.del  hat 
## Experiment 1                1.20   0.41   0.16  1.07     0.28  62.11 0.11 
## Experiment 2               -3.60  -1.39   0.75  0.43     0.09  22.72 0.11 
## Experiment 3                0.23   0.09   0.01  1.27     0.34  68.34 0.11 
## Experiment 4                0.45   0.17   0.03  1.27     0.33  67.28 0.12 
## Experiment 5                0.28   0.11   0.01  1.29     0.34  68.20 0.12 
## Experiment 6                1.13   0.41   0.16  1.10     0.28  55.98 0.12 
## Experiment 7                0.29   0.11   0.01  1.30     0.34  67.71 0.12 
## Experiment 8 (standard)    -0.78  -0.26   0.07  1.17     0.31  63.26 0.10 
## Experiment 8 (random)      -0.66  -0.22   0.05  1.19     0.32  64.94 0.10 
##                         weight  dfbs inf 
## Experiment 1             10.65  0.41     
## Experiment 2             11.07 -1.40   * 
## Experiment 3             10.78  0.08     
## Experiment 4             11.56  0.17     
## Experiment 5             11.54  0.11     
## Experiment 6             11.67  0.41     
## Experiment 7             12.18  0.11     
## Experiment 8 (standard)  10.43 -0.26     
## Experiment 8 (random)    10.12 -0.22

Experiment 2 was assessed as being an outlier.

Sensitivity analysis

A sensitivity analysis excluded Experiment 2 and re meta analysed.

results_for_meta_analysis_h2_sensitivity <- results_for_meta_analysis_h2 %>%
  filter(Experiment != "Experiment 2")

meta_results_h2_sensitivity <- 
  meta_analysis_workflow(data              = results_for_meta_analysis_h2_sensitivity,
                         effect_size_label = "Cohen's d",
                         exp_estimates     = FALSE,
                         reference_line    = 0)

• After excluding one experiment as an outlier, the meta analysed effect size was found to be of very large size (Sawilowsky, 2009): k = 8, Cohen’s d = 1.16, 95% CI = [0.91, 1.41], 95% CR = [0.52, 1.8], p < .0001, and heterogeneity was found to very negligable, Q(df = 7) = 22.72, p < .001, tau^2 = 0.09, I^2 = 75.08%, H^2 = 4.01.

H3: Behavioural intentions

results_for_meta_analysis_h3 <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", "Experiment 4", 
                            "Experiment 5", "Experiment 6", "Experiment 7"),
             logOR = c(results_exp_1_h3$logOR[results_exp_1_h3$var == "Struan"], 
                       results_exp_2_h3$logOR[results_exp_2_h3$var == "Struan"],
                       results_exp_3_h3$logOR[results_exp_3_h3$var == "Struan"],
                       results_exp_4_h3$logOR[results_exp_4_h3$var == "Struan"],
                       results_exp_5_h3$logOR[results_exp_5_h3$var == "Struan"],
                       results_exp_6_h3$logOR[results_exp_6_h3$var == "Struan"],
                       results_exp_7_h3$logOR[results_exp_7_h3$var == "Struan"]),
             logOR_se = c(results_exp_1_h3$logOR_se[results_exp_1_h3$var == "Struan"],
                          results_exp_2_h3$logOR_se[results_exp_2_h3$var == "Struan"],
                          results_exp_3_h3$logOR_se[results_exp_3_h3$var == "Struan"],
                          results_exp_4_h3$logOR_se[results_exp_4_h3$var == "Struan"],
                          results_exp_5_h3$logOR_se[results_exp_5_h3$var == "Struan"],
                          results_exp_6_h3$logOR_se[results_exp_6_h3$var == "Struan"],
                          results_exp_7_h3$logOR_se[results_exp_7_h3$var == "Struan"])) %>%
  mutate(yi  = logOR,
         sei = logOR_se)

results_for_meta_analysis_h3 %>%
  select(Experiment, yi, sei) %>%
  write_csv("../meta analysis data/results_for_meta_analysis_behavioural_intentions.csv")

meta_results_h3 <- meta_analysis_workflow(data = results_for_meta_analysis_h3,
                                          effect_size_label = "Odds Ratio",
                                          exp_estimates     = TRUE,
                                          reference_line    = 0)

• Results demonstrated a significant effect of medium size (Chen, Cohen, & Chen, 2010): k = 7, Odds Ratio = 4.71, 95% CI = [1.7, 13.2], 95% CR = [0.31, 72.97], p = .0031
• Results were found to contain a high degree of heterogeneity: Q(df = 6) = 42.25, p = 0.003, tau^2 = 1.67, I^2 = 89.78%, H^2 = 9.79

Outlier tests

influence_meta_results_h3 <- influence(meta_results_h3$fitted_model)

print(influence_meta_results_h3, digits = 2) 

## 
##              rstudent dffits cook.d cov.r tau2.del QE.del  hat weight  dfbs inf 
## Experiment 1     0.75   0.29   0.09  1.24     1.83  39.97 0.13  12.83  0.29     
## Experiment 2    -4.78  -1.67   0.67  0.22     0.16   9.31 0.14  13.81 -1.86   * 
## Experiment 3     0.27   0.11   0.01  1.37     2.03  41.95 0.13  13.31  0.11     
## Experiment 4     0.35   0.14   0.02  1.39     2.03  41.27 0.15  14.61  0.14     
## Experiment 5     0.04   0.02   0.00  1.42     2.09  42.25 0.14  14.38  0.02     
## Experiment 6     0.90   0.38   0.15  1.22     1.74  32.68 0.15  15.10  0.38     
## Experiment 7    -0.06  -0.03   0.00  1.46     2.12  41.59 0.16  15.95 -0.03

Experiment 2 was assessed as being an outlier.

Sensitivity analysis

A sensitivity analysis excluded Experiment 2 and re meta analysed.

results_for_meta_analysis_h3_sensitivity <- results_for_meta_analysis_h3 %>%
  filter(Experiment != "Experiment 2")

meta_results_h3_sensitivity <- 
  meta_analysis_workflow(data              = results_for_meta_analysis_h3_sensitivity,
                         effect_size_label = "Odds Ratio",
                         exp_estimates     = TRUE,
                         reference_line    = 0)

• After excluding one experiment as an outlier, the meta analysed effect size was found to be of large size: k = 6, Odds Ratio = 7.24, 95% CI = [4.48, 11.7], 95% CR = [2.92, 17.99], p < .0001, and heterogeneity was found to be negligable, Q(df = 5) = 9.31, p < .001, tau^2 = 0.16, I^2 = 46.44%, H^2 = 1.87.

Robustness tests via meta analyses

H1: IAT

Refit meta analyses after excluding subsets of participants based on the exploratory awareness variables.

Note that these robustness tests also take the influence tests into account by excluding Experiment 2.

# re run tests with desired subsets of participants
## exp 1
results_exp_1_h1_us_contingency_memory <- data_exp_1 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h1_color_valence_contingency_memory <- data_exp_1 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h1_demand_compliance <- data_exp_1 %>%
  filter(demand_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h1_hypothesis_aware <- data_exp_1 %>%
  filter(hypothesis_awareness == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h1_reactance_IAT <- data_exp_1 %>%
  filter(reactance_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 2
results_exp_2_h1_us_contingency_memory <- data_exp_2 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h1_color_valence_contingency_memory <- data_exp_2 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h1_demand_compliance <- data_exp_2 %>%
  filter(demand_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h1_hypothesis_aware <- data_exp_2 %>%
  filter(hypothesis_awareness == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h1_reactance_IAT <- data_exp_2 %>%
  filter(reactance_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 3
results_exp_3_h1_us_contingency_memory <- data_exp_3 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h1_color_valence_contingency_memory <- data_exp_3 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h1_demand_compliance <- data_exp_3 %>%
  filter(demand_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h1_hypothesis_aware <- data_exp_3 %>%
  filter(hypothesis_awareness == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h1_reactance_IAT <- data_exp_3 %>%
  filter(reactance_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 4
results_exp_4_h1_us_contingency_memory <- data_exp_4 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h1_color_valence_contingency_memory <- data_exp_4 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h1_demand_compliance <- data_exp_4 %>%
  filter(demand_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h1_hypothesis_aware <- data_exp_4 %>%
  filter(hypothesis_awareness == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h1_reactance_IAT <- data_exp_4 %>%
  filter(reactance_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 5
results_exp_5_h1_us_contingency_memory <- data_exp_5 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h1_color_valence_contingency_memory <- data_exp_5 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h1_demand_compliance <- data_exp_5 %>%
  filter(demand_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h1_hypothesis_aware <- data_exp_5 %>%
  filter(hypothesis_awareness == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h1_reactance_IAT <- data_exp_5 %>%
  filter(reactance_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 6
results_exp_6_h1_demand_compliance <- data_exp_6 %>%
  filter(demand_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_6_h1_hypothesis_aware <- data_exp_6 %>%
  filter(hypothesis_awareness == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_6_h1_reactance_IAT <- data_exp_6 %>%
  filter(reactance_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)


## exp 8 standard instructions
results_exp_8nri_h1_us_contingency_memory <- data_exp_8nri %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_8nri_h1_demand_compliance <- data_exp_8nri %>%
  filter(demand_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_8nri_h1_hypothesis_aware <- data_exp_8nri %>%
  filter(hypothesis_awareness == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_8nri_h1_reactance_IAT <- data_exp_8nri %>%
  filter(reactance_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)


## exp 8 random instructions
results_exp_8ri_h1_us_contingency_memory <- data_exp_8ri %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_8ri_h1_demand_compliance <- data_exp_8ri %>%
  filter(demand_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

# results_exp_8ri_h1_hypothesis_aware <- data_exp_8ri %>%
#   filter(hypothesis_awareness == FALSE) %>%
#   mutate(DV = IAT_Score,
#          IV = source_valence_condition) %>%
#   t_test_analyses(.)
# # > not enough 'x' observations

results_exp_8ri_h1_reactance_IAT <- data_exp_8ri %>%
  filter(reactance_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

# US contingency_memory
results_for_meta_analysis_h1_us_contingency_memory <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5",
                            "Experiment 8 (standard)", 
                            "Experiment 8 (random)"), 
             cohens_d = c(results_exp_1_h1_us_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_2_h1_us_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_3_h1_us_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_4_h1_us_contingency_memory$cohens_d$estimate[[1]],
                          results_exp_5_h1_us_contingency_memory$cohens_d$estimate[[1]],
                          results_exp_8nri_h1_us_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_8ri_h1_us_contingency_memory$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h1_us_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_2_h1_us_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_3_h1_us_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_4_h1_us_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_5_h1_us_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_8nri_h1_us_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_8ri_h1_us_contingency_memory$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h1_us_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_2_h1_us_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_3_h1_us_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_4_h1_us_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_5_h1_us_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_8nri_h1_us_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_8ri_h1_us_contingency_memory$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2")

# color valence contingency_memory
results_for_meta_analysis_h1_color_valence_contingency_memory <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5"), 
             cohens_d = c(results_exp_1_h1_color_valence_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_2_h1_color_valence_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_3_h1_color_valence_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_4_h1_color_valence_contingency_memory$cohens_d$estimate[[1]],
                          results_exp_5_h1_color_valence_contingency_memory$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h1_color_valence_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_2_h1_color_valence_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_3_h1_color_valence_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_4_h1_color_valence_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_5_h1_color_valence_contingency_memory$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h1_color_valence_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_2_h1_color_valence_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_3_h1_color_valence_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_4_h1_color_valence_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_5_h1_color_valence_contingency_memory$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2")

# demand_compliance
results_for_meta_analysis_h1_demand_compliance <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", "Experiment 6",
                            "Experiment 8 (standard)", 
                            "Experiment 8 (random)"), 
             cohens_d = c(results_exp_1_h1_demand_compliance$cohens_d$estimate[[1]], 
                          results_exp_2_h1_demand_compliance$cohens_d$estimate[[1]], 
                          results_exp_3_h1_demand_compliance$cohens_d$estimate[[1]], 
                          results_exp_4_h1_demand_compliance$cohens_d$estimate[[1]],
                          results_exp_5_h1_demand_compliance$cohens_d$estimate[[1]],
                          results_exp_6_h1_demand_compliance$cohens_d$estimate[[1]],
                          results_exp_8nri_h1_demand_compliance$cohens_d$estimate[[1]], 
                          results_exp_8ri_h1_demand_compliance$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h1_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_2_h1_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_3_h1_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_4_h1_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_5_h1_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_6_h1_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_8nri_h1_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_8ri_h1_demand_compliance$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h1_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_2_h1_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_3_h1_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_4_h1_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_5_h1_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_6_h1_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_8nri_h1_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_8ri_h1_demand_compliance$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2")

# hypothesis_aware
results_for_meta_analysis_h1_hypothesis_aware <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", "Experiment 6",
                            "Experiment 8 (standard)"), 
             #"Experiment 8 (random)"), # not enough observations
             cohens_d = c(results_exp_1_h1_hypothesis_aware$cohens_d$estimate[[1]], 
                          results_exp_2_h1_hypothesis_aware$cohens_d$estimate[[1]], 
                          results_exp_3_h1_hypothesis_aware$cohens_d$estimate[[1]], 
                          results_exp_4_h1_hypothesis_aware$cohens_d$estimate[[1]],
                          results_exp_5_h1_hypothesis_aware$cohens_d$estimate[[1]],
                          results_exp_6_h1_hypothesis_aware$cohens_d$estimate[[1]],
                          results_exp_8nri_h1_hypothesis_aware$cohens_d$estimate[[1]]), 
             #results_exp_8ri_h1_hypothesis_aware$cohens_d$estimate[[1]]), # not enough observations
             ci_lwr = c(results_exp_1_h1_hypothesis_aware$cohens_d$conf.int[["lower"]],
                        results_exp_2_h1_hypothesis_aware$cohens_d$conf.int[["lower"]],
                        results_exp_3_h1_hypothesis_aware$cohens_d$conf.int[["lower"]],
                        results_exp_4_h1_hypothesis_aware$cohens_d$conf.int[["lower"]],
                        results_exp_5_h1_hypothesis_aware$cohens_d$conf.int[["lower"]],
                        results_exp_6_h1_hypothesis_aware$cohens_d$conf.int[["lower"]],
                        results_exp_8nri_h1_hypothesis_aware$cohens_d$conf.int[["lower"]]),
             #results_exp_8ri_h1_hypothesis_aware$cohens_d$conf.int[["lower"]]), # not enough observations
             ci_upr = c(results_exp_1_h1_hypothesis_aware$cohens_d$conf.int[["upper"]],
                        results_exp_2_h1_hypothesis_aware$cohens_d$conf.int[["upper"]],
                        results_exp_3_h1_hypothesis_aware$cohens_d$conf.int[["upper"]],
                        results_exp_4_h1_hypothesis_aware$cohens_d$conf.int[["upper"]],
                        results_exp_5_h1_hypothesis_aware$cohens_d$conf.int[["upper"]],
                        results_exp_6_h1_hypothesis_aware$cohens_d$conf.int[["upper"]],
                        results_exp_8nri_h1_hypothesis_aware$cohens_d$conf.int[["upper"]])) %>% 
  #results_exp_8ri_h1_hypothesis_aware$cohens_d$conf.int[["upper"]])) %>%  # not enough observations
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2")

# reactance
results_for_meta_analysis_h1_reactance_IAT <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", "Experiment 6",
                            "Experiment 8 (standard)", 
                            "Experiment 8 (random)"), 
             cohens_d = c(results_exp_1_h1_reactance_IAT$cohens_d$estimate[[1]], 
                          results_exp_2_h1_reactance_IAT$cohens_d$estimate[[1]], 
                          results_exp_3_h1_reactance_IAT$cohens_d$estimate[[1]], 
                          results_exp_4_h1_reactance_IAT$cohens_d$estimate[[1]],
                          results_exp_5_h1_reactance_IAT$cohens_d$estimate[[1]],
                          results_exp_6_h1_reactance_IAT$cohens_d$estimate[[1]],
                          results_exp_8nri_h1_reactance_IAT$cohens_d$estimate[[1]], 
                          results_exp_8ri_h1_reactance_IAT$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h1_reactance_IAT$cohens_d$conf.int[["lower"]],
                        results_exp_2_h1_reactance_IAT$cohens_d$conf.int[["lower"]],
                        results_exp_3_h1_reactance_IAT$cohens_d$conf.int[["lower"]],
                        results_exp_4_h1_reactance_IAT$cohens_d$conf.int[["lower"]],
                        results_exp_5_h1_reactance_IAT$cohens_d$conf.int[["lower"]],
                        results_exp_6_h1_reactance_IAT$cohens_d$conf.int[["lower"]],
                        results_exp_8nri_h1_reactance_IAT$cohens_d$conf.int[["lower"]],
                        results_exp_8ri_h1_reactance_IAT$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h1_reactance_IAT$cohens_d$conf.int[["upper"]],
                        results_exp_2_h1_reactance_IAT$cohens_d$conf.int[["upper"]],
                        results_exp_3_h1_reactance_IAT$cohens_d$conf.int[["upper"]],
                        results_exp_4_h1_reactance_IAT$cohens_d$conf.int[["upper"]],
                        results_exp_5_h1_reactance_IAT$cohens_d$conf.int[["upper"]],
                        results_exp_6_h1_reactance_IAT$cohens_d$conf.int[["upper"]],
                        results_exp_8nri_h1_reactance_IAT$cohens_d$conf.int[["upper"]],
                        results_exp_8ri_h1_reactance_IAT$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2")

# fit metas
meta_results_h1_us_contingency_memory <- 
  meta_analysis_workflow(data              = results_for_meta_analysis_h1_us_contingency_memory,
                         effect_size_label = "Cohen's d",
                         exp_estimates     = FALSE,
                         reference_line    = 0,
                         plot = FALSE)

meta_results_h1_color_valence_contingency_memory <- 
  meta_analysis_workflow(data              = results_for_meta_analysis_h1_color_valence_contingency_memory,
                         effect_size_label = "Cohen's d",
                         exp_estimates     = FALSE,
                         reference_line    = 0,
                         plot = FALSE)

meta_results_h1_demand_compliance <- 
  meta_analysis_workflow(data              = results_for_meta_analysis_h1_demand_compliance,
                         effect_size_label = "Cohen's d",
                         exp_estimates     = FALSE,
                         reference_line    = 0,
                         plot = FALSE)

meta_results_h1_hypothesis_aware <- 
  meta_analysis_workflow(data              = results_for_meta_analysis_h1_hypothesis_aware,
                         effect_size_label = "Cohen's d",
                         exp_estimates     = FALSE,
                         reference_line    = 0,
                         plot = FALSE)

meta_results_h1_reactance_IAT <- 
  meta_analysis_workflow(data              = results_for_meta_analysis_h1_reactance_IAT,
                         effect_size_label = "Cohen's d",
                         exp_estimates     = FALSE,
                         reference_line    = 0,
                         plot = FALSE)

# robustness strings
robustness_us_contingency_memory_h1 <-
  ifelse(robust_test(meta_results_h1_us_contingency_memory$meta_analysis_results, 
                     meta_results_h1$meta_analysis_results),
         "were", "were not")

robustness_color_valence_contingency_memory_h1 <-
  ifelse(robust_test(meta_results_h1_color_valence_contingency_memory$meta_analysis_results, 
                     meta_results_h1$meta_analysis_results),
         "were", "were not")

robustness_demand_compliance_h1  <-
  ifelse(robust_test(meta_results_h1_demand_compliance$meta_analysis_results, 
                     meta_results_h1$meta_analysis_results),
         "were", "were not")

robustness_hypothesis_aware_h1   <-
  ifelse(robust_test(meta_results_h1_hypothesis_aware$meta_analysis_results, 
                     meta_results_h1$meta_analysis_results),
         "were", "were not")

robustness_reactance_IAT_excluded_h1    <-
  ifelse(robust_test(meta_results_h1_reactance_IAT$meta_analysis_results, 
                     meta_results_h1$meta_analysis_results),
         "were", "were not")

• Across studies, results were robust to including only participants who were US contingency aware, p in subsample < .0001.
• Across studies, results were robust to including only participants who were color-valence contingency aware, p in subsample = .0016.
• Across studies, results were robust to including only participants who were not demand compliant, p in subsample < .0001.
• Across studies, results were not robust to including only participants who were not hypothesis aware, p in subsample = .1338.
• Note that this exclusion removed a particularly large proportion of participants from some studies.
• Across studies, results were robust to including only participants who were not reactant, p in subsample < .0001.

H2: Self-report ratings

# re run tests with desired subsets of participants
## exp 1
results_exp_1_h2_us_contingency_memory <- data_exp_1 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h2_color_valence_contingency_memory <- data_exp_1 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h2_demand_compliance <- data_exp_1 %>%
  filter(demand_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h2_hypothesis_aware <- data_exp_1 %>%
  filter(hypothesis_awareness == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h2_reactance_self_reports <- data_exp_1 %>%
  filter(reactance_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 2
results_exp_2_h2_us_contingency_memory <- data_exp_2 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h2_color_valence_contingency_memory <- data_exp_2 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h2_demand_compliance <- data_exp_2 %>%
  filter(demand_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h2_hypothesis_aware <- data_exp_2 %>%
  filter(hypothesis_awareness == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h2_reactance_self_reports <- data_exp_2 %>%
  filter(reactance_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 3
results_exp_3_h2_us_contingency_memory <- data_exp_3 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h2_color_valence_contingency_memory <- data_exp_3 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h2_demand_compliance <- data_exp_3 %>%
  filter(demand_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h2_hypothesis_aware <- data_exp_3 %>%
  filter(hypothesis_awareness == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h2_reactance_self_reports <- data_exp_3 %>%
  filter(reactance_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 4
results_exp_4_h2_us_contingency_memory <- data_exp_4 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h2_color_valence_contingency_memory <- data_exp_4 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h2_demand_compliance <- data_exp_4 %>%
  filter(demand_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h2_hypothesis_aware <- data_exp_4 %>%
  filter(hypothesis_awareness == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h2_reactance_self_reports <- data_exp_4 %>%
  filter(reactance_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 5
results_exp_5_h2_us_contingency_memory <- data_exp_5 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h2_color_valence_contingency_memory <- data_exp_5 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h2_demand_compliance <- data_exp_5 %>%
  filter(demand_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h2_hypothesis_aware <- data_exp_5 %>%
  filter(hypothesis_awareness == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h2_reactance_self_reports <- data_exp_5 %>%
  filter(reactance_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 6
results_exp_6_h2_demand_compliance <- data_exp_6 %>%
  filter(demand_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_6_h2_hypothesis_aware <- data_exp_6 %>%
  filter(hypothesis_awareness == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_6_h2_reactance_self_reports <- data_exp_6 %>%
  filter(reactance_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)


## exp 7
results_exp_7_h2_us_contingency_memory <- data_exp_7 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_7_h2_color_valence_contingency_memory <- data_exp_7 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_7_h2_demand_compliance <- data_exp_7 %>%
  filter(demand_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_7_h2_hypothesis_aware <- data_exp_7 %>%
  filter(hypothesis_awareness == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_7_h2_reactance_self_reports <- data_exp_7 %>%
  filter(reactance_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)


## exp 8 standard instructions
results_exp_8nri_h2_us_contingency_memory <- data_exp_8nri %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_8nri_h2_demand_compliance <- data_exp_8nri %>%
  filter(demand_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_8nri_h2_hypothesis_aware <- data_exp_8nri %>%
  filter(hypothesis_awareness == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_8nri_h2_reactance_self_reports <- data_exp_8nri %>%
  filter(reactance_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)


## exp 8 random instructions
results_exp_8ri_h2_us_contingency_memory <- data_exp_8ri %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_8ri_h2_demand_compliance <- data_exp_8ri %>%
  filter(demand_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

# results_exp_8ri_h2_hypothesis_aware <- data_exp_8ri %>%
#   filter(hypothesis_awareness == FALSE) %>%
#   mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
#          IV = source_valence_condition) %>%
#   t_test_analyses(.)
# # > not enough 'x' observations

results_exp_8ri_h2_reactance_self_reports <- data_exp_8ri %>%
  filter(reactance_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)


# US contingency_memory
results_for_meta_analysis_h2_us_contingency_memory <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", 
                            "Experiment 7",
                            "Experiment 8 (standard)", 
                            "Experiment 8 (random)"), 
             cohens_d = c(results_exp_1_h2_us_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_2_h2_us_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_3_h2_us_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_4_h2_us_contingency_memory$cohens_d$estimate[[1]],
                          results_exp_5_h2_us_contingency_memory$cohens_d$estimate[[1]],
                          results_exp_7_h2_us_contingency_memory$cohens_d$estimate[[1]],
                          results_exp_8nri_h2_us_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_8ri_h2_us_contingency_memory$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h2_us_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_2_h2_us_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_3_h2_us_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_4_h2_us_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_5_h2_us_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_7_h2_us_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_8nri_h2_us_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_8ri_h2_us_contingency_memory$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h2_us_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_2_h2_us_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_3_h2_us_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_4_h2_us_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_5_h2_us_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_7_h2_us_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_8nri_h2_us_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_8ri_h2_us_contingency_memory$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2")

# color valence contingency_memory
results_for_meta_analysis_h2_color_valence_contingency_memory <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", 
                            "Experiment 7"), 
             cohens_d = c(results_exp_1_h2_color_valence_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_2_h2_color_valence_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_3_h2_color_valence_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_4_h2_color_valence_contingency_memory$cohens_d$estimate[[1]],
                          results_exp_5_h2_color_valence_contingency_memory$cohens_d$estimate[[1]],
                          results_exp_7_h2_color_valence_contingency_memory$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h2_color_valence_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_2_h2_color_valence_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_3_h2_color_valence_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_4_h2_color_valence_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_5_h2_color_valence_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_7_h2_color_valence_contingency_memory$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h2_color_valence_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_2_h2_color_valence_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_3_h2_color_valence_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_4_h2_color_valence_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_5_h2_color_valence_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_7_h2_color_valence_contingency_memory$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2")

# demand_compliance
results_for_meta_analysis_h2_demand_compliance <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", "Experiment 6",
                            "Experiment 7",
                            "Experiment 8 (standard)", 
                            "Experiment 8 (random)"), 
             cohens_d = c(results_exp_1_h2_demand_compliance$cohens_d$estimate[[1]], 
                          results_exp_2_h2_demand_compliance$cohens_d$estimate[[1]], 
                          results_exp_3_h2_demand_compliance$cohens_d$estimate[[1]], 
                          results_exp_4_h2_demand_compliance$cohens_d$estimate[[1]],
                          results_exp_5_h2_demand_compliance$cohens_d$estimate[[1]],
                          results_exp_6_h2_demand_compliance$cohens_d$estimate[[1]],
                          results_exp_7_h2_demand_compliance$cohens_d$estimate[[1]],
                          results_exp_8nri_h2_demand_compliance$cohens_d$estimate[[1]], 
                          results_exp_8ri_h2_demand_compliance$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h2_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_2_h2_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_3_h2_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_4_h2_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_5_h2_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_6_h2_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_7_h2_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_8nri_h2_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_8ri_h2_demand_compliance$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h2_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_2_h2_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_3_h2_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_4_h2_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_5_h2_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_6_h2_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_7_h2_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_8nri_h2_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_8ri_h2_demand_compliance$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2")

# hypothesis_aware
results_for_meta_analysis_h2_hypothesis_aware <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", "Experiment 6",
                            "Experiment 7",
                            "Experiment 8 (standard)"), 
             #"Experiment 8 (random)"), # not enough observations
             cohens_d = c(results_exp_1_h2_hypothesis_aware$cohens_d$estimate[[1]], 
                          results_exp_2_h2_hypothesis_aware$cohens_d$estimate[[1]], 
                          results_exp_3_h2_hypothesis_aware$cohens_d$estimate[[1]], 
                          results_exp_4_h2_hypothesis_aware$cohens_d$estimate[[1]],
                          results_exp_5_h2_hypothesis_aware$cohens_d$estimate[[1]],
                          results_exp_6_h2_hypothesis_aware$cohens_d$estimate[[1]],
                          results_exp_7_h2_hypothesis_aware$cohens_d$estimate[[1]],
                          results_exp_8nri_h2_hypothesis_aware$cohens_d$estimate[[1]]), 
             #results_exp_8ri_h2_hypothesis_aware$cohens_d$estimate[[1]]), # not enough observations
             ci_lwr = c(results_exp_1_h2_hypothesis_aware$cohens_d$conf.int[["lower"]],
                        results_exp_2_h2_hypothesis_aware$cohens_d$conf.int[["lower"]],
                        results_exp_3_h2_hypothesis_aware$cohens_d$conf.int[["lower"]],
                        results_exp_4_h2_hypothesis_aware$cohens_d$conf.int[["lower"]],
                        results_exp_5_h2_hypothesis_aware$cohens_d$conf.int[["lower"]],
                        results_exp_6_h2_hypothesis_aware$cohens_d$conf.int[["lower"]],
                        results_exp_7_h2_hypothesis_aware$cohens_d$conf.int[["lower"]],
                        results_exp_8nri_h2_hypothesis_aware$cohens_d$conf.int[["lower"]]),
             #results_exp_8ri_h2_hypothesis_aware$cohens_d$conf.int[["lower"]]), # not enough observations
             ci_upr = c(results_exp_1_h2_hypothesis_aware$cohens_d$conf.int[["upper"]],
                        results_exp_2_h2_hypothesis_aware$cohens_d$conf.int[["upper"]],
                        results_exp_3_h2_hypothesis_aware$cohens_d$conf.int[["upper"]],
                        results_exp_4_h2_hypothesis_aware$cohens_d$conf.int[["upper"]],
                        results_exp_5_h2_hypothesis_aware$cohens_d$conf.int[["upper"]],
                        results_exp_6_h2_hypothesis_aware$cohens_d$conf.int[["upper"]],
                        results_exp_7_h2_hypothesis_aware$cohens_d$conf.int[["upper"]],  
                        results_exp_8nri_h2_hypothesis_aware$cohens_d$conf.int[["upper"]])) %>%
  #results_exp_8ri_h2_hypothesis_aware$cohens_d$conf.int[["upper"]])  # not enough observations
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2")

# reactance_self_reports
results_for_meta_analysis_h2_reactance_self_reports <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", "Experiment 6",
                            "Experiment 7",
                            "Experiment 8 (standard)", 
                            "Experiment 8 (random)"), 
             cohens_d = c(results_exp_1_h2_reactance_self_reports$cohens_d$estimate[[1]], 
                          results_exp_2_h2_reactance_self_reports$cohens_d$estimate[[1]], 
                          results_exp_3_h2_reactance_self_reports$cohens_d$estimate[[1]], 
                          results_exp_4_h2_reactance_self_reports$cohens_d$estimate[[1]],
                          results_exp_5_h2_reactance_self_reports$cohens_d$estimate[[1]],
                          results_exp_6_h2_reactance_self_reports$cohens_d$estimate[[1]],
                          results_exp_7_h2_reactance_self_reports$cohens_d$estimate[[1]],
                          results_exp_8nri_h2_reactance_self_reports$cohens_d$estimate[[1]], 
                          results_exp_8ri_h2_reactance_self_reports$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h2_reactance_self_reports$cohens_d$conf.int[["lower"]],
                        results_exp_2_h2_reactance_self_reports$cohens_d$conf.int[["lower"]],
                        results_exp_3_h2_reactance_self_reports$cohens_d$conf.int[["lower"]],
                        results_exp_4_h2_reactance_self_reports$cohens_d$conf.int[["lower"]],
                        results_exp_5_h2_reactance_self_reports$cohens_d$conf.int[["lower"]],
                        results_exp_6_h2_reactance_self_reports$cohens_d$conf.int[["lower"]],
                        results_exp_7_h2_reactance_self_reports$cohens_d$conf.int[["lower"]],
                        results_exp_8nri_h2_reactance_self_reports$cohens_d$conf.int[["lower"]],
                        results_exp_8ri_h2_reactance_self_reports$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h2_reactance_self_reports$cohens_d$conf.int[["upper"]],
                        results_exp_2_h2_reactance_self_reports$cohens_d$conf.int[["upper"]],
                        results_exp_3_h2_reactance_self_reports$cohens_d$conf.int[["upper"]],
                        results_exp_4_h2_reactance_self_reports$cohens_d$conf.int[["upper"]],
                        results_exp_5_h2_reactance_self_reports$cohens_d$conf.int[["upper"]],
                        results_exp_6_h2_reactance_self_reports$cohens_d$conf.int[["upper"]],
                        results_exp_7_h2_reactance_self_reports$cohens_d$conf.int[["upper"]],
                        results_exp_8nri_h2_reactance_self_reports$cohens_d$conf.int[["upper"]],
                        results_exp_8ri_h2_reactance_self_reports$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2")

# fit metas
meta_results_h2_us_contingency_memory <- 
  meta_analysis_workflow(data              = results_for_meta_analysis_h2_us_contingency_memory,
                         effect_size_label = "Cohen's d",
                         exp_estimates     = FALSE,
                         reference_line    = 0,
                         plot = FALSE)

meta_results_h2_color_valence_contingency_memory <- 
  meta_analysis_workflow(data              = results_for_meta_analysis_h2_color_valence_contingency_memory,
                         effect_size_label = "Cohen's d",
                         exp_estimates     = FALSE,
                         reference_line    = 0,
                         plot = FALSE)

meta_results_h2_demand_compliance <- 
  meta_analysis_workflow(data              = results_for_meta_analysis_h2_demand_compliance,
                         effect_size_label = "Cohen's d",
                         exp_estimates     = FALSE,
                         reference_line    = 0,
                         plot = FALSE)

meta_results_h2_hypothesis_aware <- 
  meta_analysis_workflow(data              = results_for_meta_analysis_h2_hypothesis_aware,
                         effect_size_label = "Cohen's d",
                         exp_estimates     = FALSE,
                         reference_line    = 0,
                         plot = FALSE)

meta_results_h2_reactance_self_reports <- 
  meta_analysis_workflow(data              = results_for_meta_analysis_h2_reactance_self_reports,
                         effect_size_label = "Cohen's d",
                         exp_estimates     = FALSE,
                         reference_line    = 0,
                         plot = FALSE)

# robustness strings
robustness_us_contingency_memory_h2 <-
  ifelse(robust_test(meta_results_h2_us_contingency_memory$meta_analysis_results, 
                     meta_results_h2$meta_analysis_results),
         "were", "were not")

robustness_color_valence_contingency_memory_h2 <-
  ifelse(robust_test(meta_results_h2_color_valence_contingency_memory$meta_analysis_results, 
                     meta_results_h2$meta_analysis_results),
         "were", "were not")

robustness_demand_compliance_h2  <-
  ifelse(robust_test(meta_results_h2_demand_compliance$meta_analysis_results, 
                     meta_results_h2$meta_analysis_results),
         "were", "were not")

robustness_hypothesis_aware_h2   <-
  ifelse(robust_test(meta_results_h2_hypothesis_aware$meta_analysis_results, 
                     meta_results_h2$meta_analysis_results),
         "were", "were not")

robustness_reactance_self_reports_h2    <-
  ifelse(robust_test(meta_results_h2_reactance_self_reports$meta_analysis_results, 
                     meta_results_h2$meta_analysis_results),
         "were", "were not")

• Across studies, results were robust to including only participants who were US contingency aware, p in subsample < .0001.
• Across studies, results were robust to including only participants who were color-valence contingency aware, p in subsample < .0001.
• Across studies, results were robust to including only participants who were not demand compliant, p in subsample < .0001.
• Across studies, results were not robust to including only participants who were not hypothesis aware, p in subsample = .5155.
• Note that this exclusion removed a particularly large proportion of participants from some studies.
• Across studies, results were robust to including only participants who were not reactant, p in subsample < .0001.

H3: Behavioural intentions

# re run tests with desired subsets of participants
# exp 1
results_exp_1_h3_us_contingency_memory <- data_exp_1 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_1_h3_color_valence_contingency_memory <- data_exp_1 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_1_h3_hypothesis_aware <- data_exp_1 %>%
  filter(hypothesis_awareness == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

# exp 2
results_exp_2_h3_us_contingency_memory <- data_exp_2 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_2_h3_color_valence_contingency_memory <- data_exp_2 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_2_h3_hypothesis_aware <- data_exp_2 %>%
  filter(hypothesis_awareness == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

# exp 3
results_exp_3_h3_us_contingency_memory <- data_exp_3 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_3_h3_color_valence_contingency_memory <- data_exp_3 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_3_h3_hypothesis_aware <- data_exp_3 %>%
  filter(hypothesis_awareness == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

# exp 4
results_exp_4_h3_us_contingency_memory <- data_exp_4 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_4_h3_color_valence_contingency_memory <- data_exp_4 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_4_h3_hypothesis_aware <- data_exp_4 %>%
  filter(hypothesis_awareness == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

# exp 5
results_exp_5_h3_us_contingency_memory <- data_exp_5 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_5_h3_color_valence_contingency_memory <- data_exp_5 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_5_h3_hypothesis_aware <- data_exp_5 %>%
  filter(hypothesis_awareness == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

# exp 6
results_exp_6_h3_hypothesis_aware <- data_exp_6 %>%
  filter(hypothesis_awareness == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

# exp 7
results_exp_7_h3_us_contingency_memory <- data_exp_7 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_7_h3_color_valence_contingency_memory <- data_exp_7 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_7_h3_hypothesis_aware <- data_exp_7 %>%
  filter(hypothesis_awareness == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

# us contingency memory
results_for_meta_analysis_h3_us_contingency_memory <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", "Experiment 4", 
                            "Experiment 5", "Experiment 7"),
             logOR = c(results_exp_1_h3_us_contingency_memory$logOR[results_exp_1_h3_us_contingency_memory$var == "Struan"], 
                       results_exp_2_h3_us_contingency_memory$logOR[results_exp_2_h3_us_contingency_memory$var == "Struan"],
                       results_exp_3_h3_us_contingency_memory$logOR[results_exp_3_h3_us_contingency_memory$var == "Struan"],
                       results_exp_4_h3_us_contingency_memory$logOR[results_exp_4_h3_us_contingency_memory$var == "Struan"],
                       results_exp_5_h3_us_contingency_memory$logOR[results_exp_5_h3_us_contingency_memory$var == "Struan"],
                       results_exp_7_h3_us_contingency_memory$logOR[results_exp_7_h3_us_contingency_memory$var == "Struan"]),
             logOR_se = c(results_exp_1_h3_us_contingency_memory$logOR_se[results_exp_1_h3_us_contingency_memory$var == "Struan"],
                          results_exp_2_h3_us_contingency_memory$logOR_se[results_exp_2_h3_us_contingency_memory$var == "Struan"],
                          results_exp_3_h3_us_contingency_memory$logOR_se[results_exp_3_h3_us_contingency_memory$var == "Struan"],
                          results_exp_4_h3_us_contingency_memory$logOR_se[results_exp_4_h3_us_contingency_memory$var == "Struan"],
                          results_exp_5_h3_us_contingency_memory$logOR_se[results_exp_5_h3_us_contingency_memory$var == "Struan"],
                          results_exp_7_h3_us_contingency_memory$logOR_se[results_exp_7_h3_us_contingency_memory$var == "Struan"])) %>%
  mutate(yi  = logOR,
         sei = logOR_se) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2")

# color valence contingency memory
results_for_meta_analysis_h3_color_valence_contingency_memory <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", "Experiment 4", 
                            "Experiment 5", "Experiment 7"),
             logOR = c(results_exp_1_h3_color_valence_contingency_memory$logOR[results_exp_1_h3_color_valence_contingency_memory$var == "Struan"], 
                       results_exp_2_h3_color_valence_contingency_memory$logOR[results_exp_2_h3_color_valence_contingency_memory$var == "Struan"],
                       results_exp_3_h3_color_valence_contingency_memory$logOR[results_exp_3_h3_color_valence_contingency_memory$var == "Struan"],
                       results_exp_4_h3_color_valence_contingency_memory$logOR[results_exp_4_h3_color_valence_contingency_memory$var == "Struan"],
                       results_exp_5_h3_color_valence_contingency_memory$logOR[results_exp_5_h3_color_valence_contingency_memory$var == "Struan"],
                       results_exp_7_h3_color_valence_contingency_memory$logOR[results_exp_7_h3_color_valence_contingency_memory$var == "Struan"]),
             logOR_se = c(results_exp_1_h3_color_valence_contingency_memory$logOR_se[results_exp_1_h3_color_valence_contingency_memory$var == "Struan"],
                          results_exp_2_h3_color_valence_contingency_memory$logOR_se[results_exp_2_h3_color_valence_contingency_memory$var == "Struan"],
                          results_exp_3_h3_color_valence_contingency_memory$logOR_se[results_exp_3_h3_color_valence_contingency_memory$var == "Struan"],
                          results_exp_4_h3_color_valence_contingency_memory$logOR_se[results_exp_4_h3_color_valence_contingency_memory$var == "Struan"],
                          results_exp_5_h3_color_valence_contingency_memory$logOR_se[results_exp_5_h3_color_valence_contingency_memory$var == "Struan"],
                          results_exp_7_h3_color_valence_contingency_memory$logOR_se[results_exp_7_h3_color_valence_contingency_memory$var == "Struan"])) %>%
  mutate(yi  = logOR,
         sei = logOR_se) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2")

# hypothesis aware
results_for_meta_analysis_h3_hypothesis_aware <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", "Experiment 4", 
                            "Experiment 5", "Experiment 6", "Experiment 7"),
             logOR = c(results_exp_1_h3_hypothesis_aware$logOR[results_exp_1_h3_hypothesis_aware$var == "Struan"], 
                       results_exp_2_h3_hypothesis_aware$logOR[results_exp_2_h3_hypothesis_aware$var == "Struan"],
                       results_exp_3_h3_hypothesis_aware$logOR[results_exp_3_h3_hypothesis_aware$var == "Struan"],
                       results_exp_4_h3_hypothesis_aware$logOR[results_exp_4_h3_hypothesis_aware$var == "Struan"],
                       results_exp_5_h3_hypothesis_aware$logOR[results_exp_5_h3_hypothesis_aware$var == "Struan"],
                       results_exp_6_h3_hypothesis_aware$logOR[results_exp_6_h3_hypothesis_aware$var == "Struan"],
                       results_exp_7_h3_hypothesis_aware$logOR[results_exp_7_h3_hypothesis_aware$var == "Struan"]),
             logOR_se = c(results_exp_1_h3_hypothesis_aware$logOR_se[results_exp_1_h3_hypothesis_aware$var == "Struan"],
                          results_exp_2_h3_hypothesis_aware$logOR_se[results_exp_2_h3_hypothesis_aware$var == "Struan"],
                          results_exp_3_h3_hypothesis_aware$logOR_se[results_exp_3_h3_hypothesis_aware$var == "Struan"],
                          results_exp_4_h3_hypothesis_aware$logOR_se[results_exp_4_h3_hypothesis_aware$var == "Struan"],
                          results_exp_5_h3_hypothesis_aware$logOR_se[results_exp_5_h3_hypothesis_aware$var == "Struan"],
                          results_exp_6_h3_hypothesis_aware$logOR_se[results_exp_6_h3_hypothesis_aware$var == "Struan"],
                          results_exp_7_h3_hypothesis_aware$logOR_se[results_exp_7_h3_hypothesis_aware$var == "Struan"])) %>%
  mutate(yi  = logOR,
         sei = logOR_se) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2")

# meta fits
meta_results_h3_us_contingency_memory <- 
  meta_analysis_workflow(data = results_for_meta_analysis_h3_us_contingency_memory,
                         effect_size_label = "Odds Ratio",
                         exp_estimates     = TRUE,
                         reference_line    = 0,
                         plot = FALSE)

meta_results_h3_color_valence_contingency_memory <- 
  meta_analysis_workflow(data = results_for_meta_analysis_h3_color_valence_contingency_memory,
                         effect_size_label = "Odds Ratio",
                         exp_estimates     = TRUE,
                         reference_line    = 0,
                         plot = FALSE)

meta_results_h3_hypothesis_aware <- 
  meta_analysis_workflow(data = results_for_meta_analysis_h3_hypothesis_aware,
                         effect_size_label = "Odds Ratio",
                         exp_estimates     = TRUE,
                         reference_line    = 0,
                         plot = FALSE)

# robustness strings
robustness_us_contingency_memory_h3 <-
  ifelse(robust_test(meta_results_h3_us_contingency_memory$meta_analysis_results, 
                     meta_results_h1$meta_analysis_results),
         "were", "were not")

robustness_color_valence_contingency_memory_h3 <-
  ifelse(robust_test(meta_results_h3_color_valence_contingency_memory$meta_analysis_results, 
                     meta_results_h1$meta_analysis_results),
         "were", "were not")

robustness_hypothesis_aware_h3   <-
  ifelse(robust_test(meta_results_h3_hypothesis_aware$meta_analysis_results, 
                     meta_results_h1$meta_analysis_results),
         "were", "were not")

• Across studies, results were robust to including only participants who were US contingency aware, p in subsample < .0001.
• Across studies, results were robust to including only participants who were color-valence contingency aware, p in subsample < .0001.
• Across studies, results were not robust to including only participants who were not hypothesis aware, p in subsample = .9829.

Moderation analyses

Our robustness analyses exclude participants who meet certain criteria. A reviewer requested that we also calculate modration by these variables.

H1: IAT

Note that these tests also take the influence tests into account by excluding Experiment 2.

# re run tests with desired subsets of participants
## exp 1
results_exp_1_h1_us_contingency_memory_excluded <- data_exp_1 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h1_color_valence_contingency_memory_excluded <- data_exp_1 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h1_demand_compliance_excluded <- data_exp_1 %>%
  filter(demand_IAT == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h1_hypothesis_aware_excluded <- data_exp_1 %>%
  filter(hypothesis_awareness == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h1_reactance_IAT_excluded <- data_exp_1 %>%
  filter(reactance_IAT == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 2
results_exp_2_h1_us_contingency_memory_excluded <- data_exp_2 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h1_color_valence_contingency_memory_excluded <- data_exp_2 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h1_demand_compliance_excluded <- data_exp_2 %>%
  filter(demand_IAT == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h1_hypothesis_aware_excluded <- data_exp_2 %>%
  filter(hypothesis_awareness == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h1_reactance_IAT_excluded <- data_exp_2 %>%
  filter(reactance_IAT == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 3
results_exp_3_h1_us_contingency_memory_excluded <- data_exp_3 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h1_color_valence_contingency_memory_excluded <- data_exp_3 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h1_demand_compliance_excluded <- data_exp_3 %>%
  filter(demand_IAT == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h1_hypothesis_aware_excluded <- data_exp_3 %>%
  filter(hypothesis_awareness == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h1_reactance_IAT_excluded <- data_exp_3 %>%
  filter(reactance_IAT == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 4
results_exp_4_h1_us_contingency_memory_excluded <- data_exp_4 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h1_color_valence_contingency_memory_excluded <- data_exp_4 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h1_demand_compliance_excluded <- data_exp_4 %>%
  filter(demand_IAT == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h1_hypothesis_aware_excluded <- data_exp_4 %>%
  filter(hypothesis_awareness == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h1_reactance_IAT_excluded <- data_exp_4 %>%
  filter(reactance_IAT == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 5
results_exp_5_h1_us_contingency_memory_excluded <- data_exp_5 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h1_color_valence_contingency_memory_excluded <- data_exp_5 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h1_demand_compliance_excluded <- data_exp_5 %>%
  filter(demand_IAT == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h1_hypothesis_aware_excluded <- data_exp_5 %>%
  filter(hypothesis_awareness == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h1_reactance_IAT_excluded <- data_exp_5 %>%
  filter(reactance_IAT == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

# ## exp 6
# results_exp_6_h1_demand_compliance_excluded <- data_exp_6 %>%
#   filter(demand_IAT == TRUE) %>%
#   mutate(DV = IAT_Score,
#          IV = source_valence_condition) %>%
#   t_test_analyses(.)
#>not enough 'x' observations

results_exp_6_h1_hypothesis_aware_excluded <- data_exp_6 %>%
  filter(hypothesis_awareness == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_6_h1_reactance_IAT_excluded <- data_exp_6 %>%
  filter(reactance_IAT == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)


## exp 8 standard instructions
# results_exp_8nri_h1_us_contingency_memory_excluded <- data_exp_8nri %>%
#   filter(US_contingency_memory == FALSE) %>%
#   mutate(DV = IAT_Score,
#          IV = source_valence_condition) %>%
#   t_test_analyses(.)
#>no observations

# results_exp_8nri_h1_demand_compliance_excluded <- data_exp_8nri %>%
#   filter(demand_IAT == TRUE) %>%
#   mutate(DV = IAT_Score,
#          IV = source_valence_condition) %>%
#   t_test_analyses(.)
#>no observations

results_exp_8nri_h1_hypothesis_aware_excluded <- data_exp_8nri %>%
  filter(hypothesis_awareness == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_8nri_h1_reactance_IAT_excluded <- data_exp_8nri %>%
  filter(reactance_IAT == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)


## exp 8 random instructions
results_exp_8ri_h1_us_contingency_memory_excluded <- data_exp_8ri %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

# results_exp_8ri_h1_demand_compliance_excluded <- data_exp_8ri %>%
#   filter(demand_IAT == TRUE) %>%
#   mutate(DV = IAT_Score,
#          IV = source_valence_condition) %>%
#   t_test_analyses(.)
# # no observations

results_exp_8ri_h1_hypothesis_aware_excluded <- data_exp_8ri %>%
  filter(hypothesis_awareness == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_8ri_h1_reactance_IAT_excluded <- data_exp_8ri %>%
  filter(reactance_IAT == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

# US contingency_memory
results_for_meta_analysis_h1_us_contingency_memory_excluded <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5",
                            #"Experiment 8 (standard)", 
                            "Experiment 8 (random)"),
             cohens_d = c(results_exp_1_h1_us_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_2_h1_us_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_3_h1_us_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_4_h1_us_contingency_memory_excluded$cohens_d$estimate[[1]],
                          results_exp_5_h1_us_contingency_memory_excluded$cohens_d$estimate[[1]],
                          #results_exp_8nri_h1_us_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_8ri_h1_us_contingency_memory_excluded$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h1_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_2_h1_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_3_h1_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_4_h1_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_5_h1_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        #results_exp_8nri_h1_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_8ri_h1_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h1_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_2_h1_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_3_h1_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_4_h1_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_5_h1_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        #results_exp_8nri_h1_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_8ri_h1_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2") %>%
  mutate(exclude = TRUE)

# color valence contingency_memory
results_for_meta_analysis_h1_color_valence_contingency_memory_excluded <- 
  data.frame(Experiment = c("Experiment 1", 
                            "Experiment 2", 
                            "Experiment 3", 
                            "Experiment 4", 
                            "Experiment 5"), 
             cohens_d = c(results_exp_1_h1_color_valence_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_2_h1_color_valence_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_3_h1_color_valence_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_4_h1_color_valence_contingency_memory_excluded$cohens_d$estimate[[1]],
                          results_exp_5_h1_color_valence_contingency_memory_excluded$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h1_color_valence_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_2_h1_color_valence_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_3_h1_color_valence_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_4_h1_color_valence_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_5_h1_color_valence_contingency_memory_excluded$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h1_color_valence_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_2_h1_color_valence_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_3_h1_color_valence_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_4_h1_color_valence_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_5_h1_color_valence_contingency_memory_excluded$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2") %>%
  mutate(exclude = TRUE)

# demand_compliance
results_for_meta_analysis_h1_demand_compliance_excluded <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5"), 
             #"Experiment 6",
             #"Experiment 8 (standard)"), 
             #"Experiment 8 (random)"), 
             cohens_d = c(results_exp_1_h1_demand_compliance_excluded$cohens_d$estimate[[1]], 
                          results_exp_2_h1_demand_compliance_excluded$cohens_d$estimate[[1]], 
                          results_exp_3_h1_demand_compliance_excluded$cohens_d$estimate[[1]], 
                          results_exp_4_h1_demand_compliance_excluded$cohens_d$estimate[[1]],
                          results_exp_5_h1_demand_compliance_excluded$cohens_d$estimate[[1]]),
             #results_exp_6_h1_demand_compliance_excluded_excluded$cohens_d$estimate[[1]],
             #results_exp_8nri_h1_demand_compliance_excluded$cohens_d$estimate[[1]]), 
             #results_exp_8ri_h1_demand_compliance_excluded$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h1_demand_compliance_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_2_h1_demand_compliance_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_3_h1_demand_compliance_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_4_h1_demand_compliance_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_5_h1_demand_compliance_excluded$cohens_d$conf.int[["lower"]]),
             #results_exp_6_h1_demand_compliance_excluded$cohens_d$conf.int[["lower"]],
             #results_exp_8nri_h1_demand_compliance_excluded$cohens_d$conf.int[["lower"]]),
             #results_exp_8ri_h1_demand_compliance_excluded$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h1_demand_compliance_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_2_h1_demand_compliance_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_3_h1_demand_compliance_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_4_h1_demand_compliance_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_5_h1_demand_compliance_excluded$cohens_d$conf.int[["upper"]])) %>%
  #results_exp_6_h1_demand_compliance_excluded$cohens_d_excluded$conf.int[["upper"]],
  #results_exp_8nri_h1_demand_compliance_excluded$cohens_d$conf.int[["upper"]])) %>%
  #results_exp_8ri_h1_demand_compliance_excluded$cohens_d_excluded$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2") %>%
  mutate(exclude = TRUE)

# hypothesis_aware
results_for_meta_analysis_h1_hypothesis_aware_excluded <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", "Experiment 6"),
             #"Experiment 8 (standard)"),  # not enough observations
             #"Experiment 8 (random)"), 
             cohens_d = c(results_exp_1_h1_hypothesis_aware_excluded$cohens_d$estimate[[1]], 
                          results_exp_2_h1_hypothesis_aware_excluded$cohens_d$estimate[[1]], 
                          results_exp_3_h1_hypothesis_aware_excluded$cohens_d$estimate[[1]], 
                          results_exp_4_h1_hypothesis_aware_excluded$cohens_d$estimate[[1]],
                          results_exp_5_h1_hypothesis_aware_excluded$cohens_d$estimate[[1]],
                          results_exp_6_h1_hypothesis_aware_excluded$cohens_d$estimate[[1]]),
             #results_exp_8nri_h1_hypothesis_aware$cohens_d_excluded$estimate[[1]]), 
             #results_exp_8ri_h1_hypothesis_aware$cohens_d_excluded$estimate[[1]]), # not enough observations
             ci_lwr = c(results_exp_1_h1_hypothesis_aware_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_2_h1_hypothesis_aware_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_3_h1_hypothesis_aware_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_4_h1_hypothesis_aware_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_5_h1_hypothesis_aware_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_6_h1_hypothesis_aware_excluded$cohens_d$conf.int[["lower"]]),
             #results_exp_8nri_h1_hypothesis_aware_excluded$cohens_d$conf.int[["lower"]]),
             #results_exp_8ri_h1_hypothesis_aware_excluded$cohens_d$conf.int[["lower"]]), # not enough observations
             ci_upr = c(results_exp_1_h1_hypothesis_aware_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_2_h1_hypothesis_aware_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_3_h1_hypothesis_aware_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_4_h1_hypothesis_aware_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_5_h1_hypothesis_aware_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_6_h1_hypothesis_aware_excluded$cohens_d$conf.int[["upper"]])) %>% 
  #results_exp_8nri_h1_hypothesis_aware_excluded$cohens_d$conf.int[["upper"]])) %>% 
  #results_exp_8ri_h1_hypothesis_aware_excluded$cohens_d$conf.int[["upper"]])) %>%  # not enough observations
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2") %>%
  mutate(exclude = TRUE)

# reactance
results_for_meta_analysis_h1_reactance_IAT_excluded <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", "Experiment 6",
                            "Experiment 8 (standard)", 
                            "Experiment 8 (random)"), 
             cohens_d = c(results_exp_1_h1_reactance_IAT_excluded$cohens_d$estimate[[1]], 
                          results_exp_2_h1_reactance_IAT_excluded$cohens_d$estimate[[1]], 
                          results_exp_3_h1_reactance_IAT_excluded$cohens_d$estimate[[1]], 
                          results_exp_4_h1_reactance_IAT_excluded$cohens_d$estimate[[1]],
                          results_exp_5_h1_reactance_IAT_excluded$cohens_d$estimate[[1]],
                          results_exp_6_h1_reactance_IAT_excluded$cohens_d$estimate[[1]],
                          results_exp_8nri_h1_reactance_IAT_excluded$cohens_d$estimate[[1]], 
                          results_exp_8ri_h1_reactance_IAT_excluded$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h1_reactance_IAT_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_2_h1_reactance_IAT_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_3_h1_reactance_IAT_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_4_h1_reactance_IAT_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_5_h1_reactance_IAT_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_6_h1_reactance_IAT_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_8nri_h1_reactance_IAT_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_8ri_h1_reactance_IAT_excluded$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h1_reactance_IAT_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_2_h1_reactance_IAT_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_3_h1_reactance_IAT_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_4_h1_reactance_IAT_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_5_h1_reactance_IAT_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_6_h1_reactance_IAT_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_8nri_h1_reactance_IAT_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_8ri_h1_reactance_IAT_excluded$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2") %>%
  mutate(exclude = TRUE)

# fit metas
mod_meta_results_h1_us_contingency_memory <- 
  moderator_meta_analysis_workflow(data = bind_rows(semi_join(mutate(results_for_meta_analysis_h1_us_contingency_memory, exclude = FALSE),
                                                              results_for_meta_analysis_h1_us_contingency_memory_excluded,
                                                              by = "Experiment"), 
                                                    results_for_meta_analysis_h1_us_contingency_memory_excluded),
                                   effect_size_label = "Cohen's d")

mod_meta_results_h1_color_valence_contingency_memory <- 
  moderator_meta_analysis_workflow(data = bind_rows(semi_join(mutate(results_for_meta_analysis_h1_color_valence_contingency_memory, exclude = FALSE),
                                                              results_for_meta_analysis_h1_color_valence_contingency_memory_excluded,
                                                              by = "Experiment"),
                                                    results_for_meta_analysis_h1_color_valence_contingency_memory_excluded),
                                   effect_size_label = "Cohen's d")

mod_meta_results_h1_demand_compliance <- 
  moderator_meta_analysis_workflow(data = bind_rows(semi_join(mutate(results_for_meta_analysis_h1_demand_compliance, exclude = FALSE),
                                                              results_for_meta_analysis_h1_demand_compliance_excluded,
                                                              by = "Experiment"),
                                                    results_for_meta_analysis_h1_demand_compliance_excluded),
                                   effect_size_label = "Cohen's d")

mod_meta_results_h1_hypothesis_aware <- 
  moderator_meta_analysis_workflow(data = bind_rows(semi_join(mutate(results_for_meta_analysis_h1_hypothesis_aware, exclude = FALSE),
                                                              results_for_meta_analysis_h1_hypothesis_aware_excluded,
                                                              by = "Experiment"), 
                                                    results_for_meta_analysis_h1_hypothesis_aware_excluded),
                                   effect_size_label = "Cohen's d")

mod_meta_results_h1_reactance <- 
  moderator_meta_analysis_workflow(data = bind_rows(semi_join(mutate(results_for_meta_analysis_h1_reactance_IAT, exclude = FALSE),
                                                              results_for_meta_analysis_h1_reactance_IAT_excluded,
                                                              by = "Experiment"),
                                                    results_for_meta_analysis_h1_reactance_IAT_excluded),
                                   effect_size_label = "Cohen's d")

# forest plots
forest(mod_meta_results_h1_us_contingency_memory$fitted_model)

forest(mod_meta_results_h1_color_valence_contingency_memory$fitted_model)

forest(mod_meta_results_h1_demand_compliance$fitted_model)

forest(mod_meta_results_h1_hypothesis_aware$fitted_model)

forest(mod_meta_results_h1_reactance$fitted_model)

Moderation by:

• US contingency memory (TRUE were excluded): QM(df = 1) = 0.01, p = 0.903, difference in excluded subset Cohen’s d = 0.03, 95% CI [-0.48, 0.54]
• color-valence contingency memory (TRUE were excluded): QM(df = 1) = 2.45, p = 0.118, difference in excluded subset Cohen’s d = -0.76, 95% CI [-1.72, 0.19]
• demand compliance (FALSE were excluded): QM(df = 1) = 0.01, p = 0.929, difference in excluded subset Cohen’s d = 0.04, 95% CI [-0.77, 0.84]
• hypothesis awareness (FALSE were excluded): QM(df = 1) = 5.34, p = 0.021, difference in excluded subset Cohen’s d = 0.73, 95% CI [0.11, 1.35]
• reactance (FALSE were excluded): QM(df = 1) = 0.01, p = 0.923, difference in excluded subset Cohen’s d = -0.03, 95% CI [-0.54, 0.49]

H2: Self-report ratings

# re run tests with desired subsets of participants
## exp 1
results_exp_1_h2_us_contingency_memory_excluded <- data_exp_1 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h2_color_valence_contingency_memory_excluded <- data_exp_1 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h2_demand_compliance_excluded <- data_exp_1 %>%
  filter(demand_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h2_hypothesis_aware_excluded <- data_exp_1 %>%
  filter(hypothesis_awareness == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h2_reactance_self_reports_excluded <- data_exp_1 %>%
  filter(reactance_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 2
results_exp_2_h2_us_contingency_memory_excluded <- data_exp_2 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h2_color_valence_contingency_memory_excluded <- data_exp_2 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h2_demand_compliance_excluded <- data_exp_2 %>%
  filter(demand_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h2_hypothesis_aware_excluded <- data_exp_2 %>%
  filter(hypothesis_awareness == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h2_reactance_self_reports_excluded <- data_exp_2 %>%
  filter(reactance_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 3
results_exp_3_h2_us_contingency_memory_excluded <- data_exp_3 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h2_color_valence_contingency_memory_excluded <- data_exp_3 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h2_demand_compliance_excluded <- data_exp_3 %>%
  filter(demand_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h2_hypothesis_aware_excluded <- data_exp_3 %>%
  filter(hypothesis_awareness == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h2_reactance_self_reports_excluded <- data_exp_3 %>%
  filter(reactance_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 4
results_exp_4_h2_us_contingency_memory_excluded <- data_exp_4 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h2_color_valence_contingency_memory_excluded <- data_exp_4 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h2_demand_compliance_excluded <- data_exp_4 %>%
  filter(demand_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h2_hypothesis_aware_excluded <- data_exp_4 %>%
  filter(hypothesis_awareness == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h2_reactance_self_reports_excluded <- data_exp_4 %>%
  filter(reactance_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 5
results_exp_5_h2_us_contingency_memory_excluded <- data_exp_5 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h2_color_valence_contingency_memory_excluded <- data_exp_5 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h2_demand_compliance_excluded <- data_exp_5 %>%
  filter(demand_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h2_hypothesis_aware_excluded <- data_exp_5 %>%
  filter(hypothesis_awareness == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h2_reactance_self_reports_excluded <- data_exp_5 %>%
  filter(reactance_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 6
results_exp_6_h2_demand_compliance_excluded <- data_exp_6 %>%
  filter(demand_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_6_h2_hypothesis_aware_excluded <- data_exp_6 %>%
  filter(hypothesis_awareness == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_6_h2_reactance_self_reports_excluded <- data_exp_6 %>%
  filter(reactance_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)


## exp 7
results_exp_7_h2_us_contingency_memory_excluded <- data_exp_7 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_7_h2_color_valence_contingency_memory_excluded <- data_exp_7 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_7_h2_demand_compliance_excluded <- data_exp_7 %>%
  filter(demand_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_7_h2_hypothesis_aware_excluded <- data_exp_7 %>%
  filter(hypothesis_awareness == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_7_h2_reactance_self_reports_excluded <- data_exp_7 %>%
  filter(reactance_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)


## exp 8 standard instructions
# results_exp_8nri_h2_us_contingency_memory_excluded <- data_exp_8nri %>%
#   filter(US_contingency_memory == FALSE) %>%
#   mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
#          IV = source_valence_condition) %>%
#   t_test_analyses(.)
# > NOT ENOUGH OBSERVATIONS

results_exp_8nri_h2_demand_compliance_excluded <- data_exp_8nri %>%
  filter(demand_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_8nri_h2_hypothesis_aware_excluded <- data_exp_8nri %>%
  filter(hypothesis_awareness == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_8nri_h2_reactance_self_reports_excluded <- data_exp_8nri %>%
  filter(reactance_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)


## exp 8 random instructions
results_exp_8ri_h2_us_contingency_memory_excluded <- data_exp_8ri %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_8ri_h2_demand_compliance_excluded <- data_exp_8ri %>%
  filter(demand_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

# results_exp_8ri_h2_hypothesis_aware_excluded <- data_exp_8ri %>%
#   filter(hypothesis_awareness == TRUE) %>%
#   mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
#          IV = source_valence_condition) %>%
#   t_test_analyses(.)
# # > not enough 'x' observations

results_exp_8ri_h2_reactance_self_reports_excluded <- data_exp_8ri %>%
  filter(reactance_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)


# US contingency_memory
results_for_meta_analysis_h2_us_contingency_memory_excluded <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", 
                            "Experiment 7",
                            #"Experiment 8 (standard)", 
                            "Experiment 8 (random)"), 
             cohens_d = c(results_exp_1_h2_us_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_2_h2_us_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_3_h2_us_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_4_h2_us_contingency_memory_excluded$cohens_d$estimate[[1]],
                          results_exp_5_h2_us_contingency_memory_excluded$cohens_d$estimate[[1]],
                          results_exp_7_h2_us_contingency_memory$cohens_d$estimate[[1]],
                          #results_exp_8nri_h2_us_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_8ri_h2_us_contingency_memory_excluded$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h2_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_2_h2_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_3_h2_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_4_h2_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_5_h2_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_7_h2_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        #results_exp_8nri_h2_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_8ri_h2_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h2_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_2_h2_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_3_h2_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_4_h2_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_5_h2_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_7_h2_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        #results_exp_8nri_h2_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_8ri_h2_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2") %>%
  mutate(exclude = TRUE)

# color valence contingency_memory
results_for_meta_analysis_h2_color_valence_contingency_memory_excluded <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", 
                            "Experiment 7"), 
             cohens_d = c(results_exp_1_h2_color_valence_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_2_h2_color_valence_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_3_h2_color_valence_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_4_h2_color_valence_contingency_memory_excluded$cohens_d$estimate[[1]],
                          results_exp_5_h2_color_valence_contingency_memory_excluded$cohens_d$estimate[[1]],
                          results_exp_7_h2_color_valence_contingency_memory_excluded$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h2_color_valence_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_2_h2_color_valence_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_3_h2_color_valence_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_4_h2_color_valence_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_5_h2_color_valence_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_7_h2_color_valence_contingency_memory_excluded$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h2_color_valence_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_2_h2_color_valence_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_3_h2_color_valence_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_4_h2_color_valence_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_5_h2_color_valence_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_7_h2_color_valence_contingency_memory_excluded$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2") %>%
  mutate(exclude = TRUE)

# demand_compliance
results_for_meta_analysis_h2_demand_compliance_excluded <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", "Experiment 6",
                            "Experiment 7",
                            "Experiment 8 (standard)", 
                            "Experiment 8 (random)"), 
             cohens_d = c(results_exp_1_h2_demand_compliance_excluded$cohens_d$estimate[[1]], 
                          results_exp_2_h2_demand_compliance_excluded$cohens_d$estimate[[1]], 
                          results_exp_3_h2_demand_compliance_excluded$cohens_d$estimate[[1]], 
                          results_exp_4_h2_demand_compliance_excluded$cohens_d$estimate[[1]],
                          results_exp_5_h2_demand_compliance_excluded$cohens_d$estimate[[1]],
                          results_exp_6_h2_demand_compliance_excluded$cohens_d$estimate[[1]],
                          results_exp_7_h2_demand_compliance_excluded$cohens_d$estimate[[1]],
                          results_exp_8nri_h2_demand_compliance_excluded$cohens_d$estimate[[1]], 
                          results_exp_8ri_h2_demand_compliance_excluded$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h2_demand_compliance_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_2_h2_demand_compliance_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_3_h2_demand_compliance_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_4_h2_demand_compliance_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_5_h2_demand_compliance_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_6_h2_demand_compliance_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_7_h2_demand_compliance_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_8nri_h2_demand_compliance_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_8ri_h2_demand_compliance_excluded$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h2_demand_compliance_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_2_h2_demand_compliance_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_3_h2_demand_compliance_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_4_h2_demand_compliance_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_5_h2_demand_compliance_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_6_h2_demand_compliance_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_7_h2_demand_compliance_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_8nri_h2_demand_compliance_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_8ri_h2_demand_compliance_excluded$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2") %>%
  mutate(exclude = TRUE)

# hypothesis_aware
results_for_meta_analysis_h2_hypothesis_aware_excluded <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", "Experiment 6",
                            "Experiment 7",
                            "Experiment 8 (standard)"), 
             #"Experiment 8 (random)"), # not enough observations
             cohens_d = c(results_exp_1_h2_hypothesis_aware_excluded$cohens_d$estimate[[1]], 
                          results_exp_2_h2_hypothesis_aware_excluded$cohens_d$estimate[[1]], 
                          results_exp_3_h2_hypothesis_aware_excluded$cohens_d$estimate[[1]], 
                          results_exp_4_h2_hypothesis_aware_excluded$cohens_d$estimate[[1]],
                          results_exp_5_h2_hypothesis_aware_excluded$cohens_d$estimate[[1]],
                          results_exp_6_h2_hypothesis_aware_excluded$cohens_d$estimate[[1]],
                          results_exp_7_h2_hypothesis_aware_excluded$cohens_d$estimate[[1]],
                          results_exp_8nri_h2_hypothesis_aware_excluded$cohens_d$estimate[[1]]), 
             #results_exp_8ri_h2_hypothesis_aware_excluded$cohens_d$estimate[[1]]), # not enough observations
             ci_lwr = c(results_exp_1_h2_hypothesis_aware_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_2_h2_hypothesis_aware_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_3_h2_hypothesis_aware_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_4_h2_hypothesis_aware_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_5_h2_hypothesis_aware_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_6_h2_hypothesis_aware_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_7_h2_hypothesis_aware_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_8nri_h2_hypothesis_aware_excluded$cohens_d$conf.int[["lower"]]),
             #results_exp_8ri_h2_hypothesis_aware_excluded$cohens_d$conf.int[["lower"]]), # not enough observations
             ci_upr = c(results_exp_1_h2_hypothesis_aware_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_2_h2_hypothesis_aware_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_3_h2_hypothesis_aware_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_4_h2_hypothesis_aware_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_5_h2_hypothesis_aware_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_6_h2_hypothesis_aware_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_7_h2_hypothesis_aware_excluded$cohens_d$conf.int[["upper"]],  
                        results_exp_8nri_h2_hypothesis_aware_excluded$cohens_d$conf.int[["upper"]])) %>%
  #results_exp_8ri_h2_hypothesis_aware_excluded$cohens_d$conf.int[["upper"]])  # not enough observations
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2") %>%
  mutate(exclude = TRUE)

# reactance_self_reports
results_for_meta_analysis_h2_reactance_self_reports_excluded <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", "Experiment 6",
                            "Experiment 7",
                            "Experiment 8 (standard)", 
                            "Experiment 8 (random)"), 
             cohens_d = c(results_exp_1_h2_reactance_self_reports_excluded$cohens_d$estimate[[1]], 
                          results_exp_2_h2_reactance_self_reports_excluded$cohens_d$estimate[[1]], 
                          results_exp_3_h2_reactance_self_reports_excluded$cohens_d$estimate[[1]], 
                          results_exp_4_h2_reactance_self_reports_excluded$cohens_d$estimate[[1]],
                          results_exp_5_h2_reactance_self_reports_excluded$cohens_d$estimate[[1]],
                          results_exp_6_h2_reactance_self_reports_excluded$cohens_d$estimate[[1]],
                          results_exp_7_h2_reactance_self_reports_excluded$cohens_d$estimate[[1]],
                          results_exp_8nri_h2_reactance_self_reports_excluded$cohens_d$estimate[[1]], 
                          results_exp_8ri_h2_reactance_self_reports_excluded$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h2_reactance_self_reports_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_2_h2_reactance_self_reports_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_3_h2_reactance_self_reports_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_4_h2_reactance_self_reports_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_5_h2_reactance_self_reports_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_6_h2_reactance_self_reports_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_7_h2_reactance_self_reports_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_8nri_h2_reactance_self_reports_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_8ri_h2_reactance_self_reports_excluded$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h2_reactance_self_reports_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_2_h2_reactance_self_reports_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_3_h2_reactance_self_reports_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_4_h2_reactance_self_reports_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_5_h2_reactance_self_reports_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_6_h2_reactance_self_reports_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_7_h2_reactance_self_reports_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_8nri_h2_reactance_self_reports_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_8ri_h2_reactance_self_reports_excluded$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2") %>%
  mutate(exclude = TRUE)

# fit metas
mod_meta_results_h2_us_contingency_memory <- 
  moderator_meta_analysis_workflow(data = bind_rows(semi_join(mutate(results_for_meta_analysis_h2_us_contingency_memory, exclude = FALSE),
                                                              results_for_meta_analysis_h2_us_contingency_memory_excluded,
                                                              by = "Experiment"), 
                                                    results_for_meta_analysis_h2_us_contingency_memory_excluded),
                                   effect_size_label = "Cohen's d")

mod_meta_results_h2_color_valence_contingency_memory <- 
  moderator_meta_analysis_workflow(data = bind_rows(semi_join(mutate(results_for_meta_analysis_h2_color_valence_contingency_memory, exclude = FALSE),
                                                              results_for_meta_analysis_h2_color_valence_contingency_memory_excluded,
                                                              by = "Experiment"),
                                                    results_for_meta_analysis_h2_color_valence_contingency_memory_excluded),
                                   effect_size_label = "Cohen's d")

mod_meta_results_h2_demand_compliance <- 
  moderator_meta_analysis_workflow(data = bind_rows(semi_join(mutate(results_for_meta_analysis_h2_demand_compliance, exclude = FALSE),
                                                              results_for_meta_analysis_h2_demand_compliance_excluded,
                                                              by = "Experiment"),
                                                    results_for_meta_analysis_h2_demand_compliance_excluded), 
                                   effect_size_label = "Cohen's d")

mod_meta_results_h2_hypothesis_aware <- 
  moderator_meta_analysis_workflow(data = bind_rows(semi_join(mutate(results_for_meta_analysis_h2_hypothesis_aware, exclude = FALSE),
                                                              results_for_meta_analysis_h2_hypothesis_aware_excluded,
                                                              by = "Experiment"),
                                                    results_for_meta_analysis_h2_hypothesis_aware_excluded), 
                                   effect_size_label = "Cohen's d")

mod_meta_results_h2_reactance_self_reports <- 
  moderator_meta_analysis_workflow(data = bind_rows(semi_join(mutate(results_for_meta_analysis_h2_reactance_self_reports, exclude = FALSE),
                                                              results_for_meta_analysis_h2_reactance_self_reports_excluded,
                                                              by = "Experiment"),
                                                    results_for_meta_analysis_h2_reactance_self_reports_excluded), 
                                   effect_size_label = "Cohen's d")

# forest plots
forest(mod_meta_results_h2_us_contingency_memory$fitted_model)

forest(mod_meta_results_h2_color_valence_contingency_memory$fitted_model)

forest(mod_meta_results_h2_demand_compliance$fitted_model)

forest(mod_meta_results_h2_hypothesis_aware$fitted_model)

forest(mod_meta_results_h2_reactance_self_reports$fitted_model)

Moderation by:

• US contingency memory (TRUE were excluded): QM(df = 1) = 0.46, p = 0.497, difference in excluded subset Cohen’s d = 0.11, 95% CI [-0.21, 0.43]
• color-valence contingency memory (TRUE were excluded): QM(df = 1) = 11.28, p < .001, difference in excluded subset Cohen’s d = -1.46, 95% CI [-2.31, -0.61]
• demand compliance (FALSE were excluded): QM(df = 1) = 9.74, p = 0.002, difference in excluded subset Cohen’s d = 0.76, 95% CI [0.28, 1.24]
• hypothesis awareness (FALSE were excluded): QM(df = 1) = 20.5, p < .001, difference in excluded subset Cohen’s d = 1.4, 95% CI [0.79, 2.01]
• reactance (FALSE were excluded): QM(df = 1) = 1.55, p = 0.213, difference in excluded subset Cohen’s d = -0.28, 95% CI [-0.71, 0.16]

H3: Behavioural intentions

# re run tests with desired subsets of participants
# exp 1
results_exp_1_h3_us_contingency_memory_excluded <- data_exp_1 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_1_h3_color_valence_contingency_memory_excluded <- data_exp_1 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_1_h3_hypothesis_aware_excluded <- data_exp_1 %>%
  filter(hypothesis_awareness == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

# exp 2
results_exp_2_h3_us_contingency_memory_excluded <- data_exp_2 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_2_h3_color_valence_contingency_memory_excluded <- data_exp_2 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_2_h3_hypothesis_aware_excluded <- data_exp_2 %>%
  filter(hypothesis_awareness == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

# exp 3
results_exp_3_h3_us_contingency_memory_excluded <- data_exp_3 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_3_h3_color_valence_contingency_memory_excluded <- data_exp_3 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_3_h3_hypothesis_aware_excluded <- data_exp_3 %>%
  filter(hypothesis_awareness == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

# exp 4
results_exp_4_h3_us_contingency_memory_excluded <- data_exp_4 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_4_h3_color_valence_contingency_memory_excluded <- data_exp_4 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_4_h3_hypothesis_aware_excluded <- data_exp_4 %>%
  filter(hypothesis_awareness == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

# exp 5
results_exp_5_h3_us_contingency_memory_excluded <- data_exp_5 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_5_h3_color_valence_contingency_memory_excluded <- data_exp_5 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_5_h3_hypothesis_aware_excluded <- data_exp_5 %>%
  filter(hypothesis_awareness == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

# exp 6
results_exp_6_h3_hypothesis_aware_excluded <- data_exp_6 %>%
  filter(hypothesis_awareness == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

# exp 7
results_exp_7_h3_us_contingency_memory_excluded <- data_exp_7 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_7_h3_color_valence_contingency_memory_excluded <- data_exp_7 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_7_h3_hypothesis_aware_excluded <- data_exp_7 %>%
  filter(hypothesis_awareness == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

# us contingency memory
results_for_meta_analysis_h3_us_contingency_memory_excluded <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", "Experiment 4", 
                            "Experiment 5", "Experiment 7"),
             logOR = c(results_exp_1_h3_us_contingency_memory_excluded$logOR[results_exp_1_h3_us_contingency_memory_excluded$var == "Struan"], 
                       results_exp_2_h3_us_contingency_memory_excluded$logOR[results_exp_2_h3_us_contingency_memory_excluded$var == "Struan"],
                       results_exp_3_h3_us_contingency_memory_excluded$logOR[results_exp_3_h3_us_contingency_memory_excluded$var == "Struan"],
                       results_exp_4_h3_us_contingency_memory_excluded$logOR[results_exp_4_h3_us_contingency_memory_excluded$var == "Struan"],
                       results_exp_5_h3_us_contingency_memory_excluded$logOR[results_exp_5_h3_us_contingency_memory_excluded$var == "Struan"],
                       results_exp_7_h3_us_contingency_memory_excluded$logOR[results_exp_7_h3_us_contingency_memory_excluded$var == "Struan"]),
             logOR_se = c(results_exp_1_h3_us_contingency_memory_excluded$logOR_se[results_exp_1_h3_us_contingency_memory_excluded$var == "Struan"],
                          results_exp_2_h3_us_contingency_memory_excluded$logOR_se[results_exp_2_h3_us_contingency_memory_excluded$var == "Struan"],
                          results_exp_3_h3_us_contingency_memory_excluded$logOR_se[results_exp_3_h3_us_contingency_memory_excluded$var == "Struan"],
                          results_exp_4_h3_us_contingency_memory_excluded$logOR_se[results_exp_4_h3_us_contingency_memory_excluded$var == "Struan"],
                          results_exp_5_h3_us_contingency_memory_excluded$logOR_se[results_exp_5_h3_us_contingency_memory_excluded$var == "Struan"],
                          results_exp_7_h3_us_contingency_memory_excluded$logOR_se[results_exp_7_h3_us_contingency_memory_excluded$var == "Struan"])) %>%
  mutate(yi  = logOR,
         sei = logOR_se) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2") %>%
  mutate(exclude = TRUE)

# color valence contingency memory
results_for_meta_analysis_h3_color_valence_contingency_memory_excluded <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", "Experiment 4", 
                            "Experiment 5", "Experiment 7"),
             logOR = c(results_exp_1_h3_color_valence_contingency_memory_excluded$logOR[results_exp_1_h3_color_valence_contingency_memory_excluded$var == "Struan"], 
                       results_exp_2_h3_color_valence_contingency_memory_excluded$logOR[results_exp_2_h3_color_valence_contingency_memory_excluded$var == "Struan"],
                       results_exp_3_h3_color_valence_contingency_memory_excluded$logOR[results_exp_3_h3_color_valence_contingency_memory_excluded$var == "Struan"],
                       results_exp_4_h3_color_valence_contingency_memory_excluded$logOR[results_exp_4_h3_color_valence_contingency_memory_excluded$var == "Struan"],
                       results_exp_5_h3_color_valence_contingency_memory_excluded$logOR[results_exp_5_h3_color_valence_contingency_memory_excluded$var == "Struan"],
                       results_exp_7_h3_color_valence_contingency_memory_excluded$logOR[results_exp_7_h3_color_valence_contingency_memory_excluded$var == "Struan"]),
             logOR_se = c(results_exp_1_h3_color_valence_contingency_memory_excluded$logOR_se[results_exp_1_h3_color_valence_contingency_memory$var == "Struan"],
                          results_exp_2_h3_color_valence_contingency_memory_excluded$logOR_se[results_exp_2_h3_color_valence_contingency_memory$var == "Struan"],
                          results_exp_3_h3_color_valence_contingency_memory_excluded$logOR_se[results_exp_3_h3_color_valence_contingency_memory$var == "Struan"],
                          results_exp_4_h3_color_valence_contingency_memory_excluded$logOR_se[results_exp_4_h3_color_valence_contingency_memory$var == "Struan"],
                          results_exp_5_h3_color_valence_contingency_memory_excluded$logOR_se[results_exp_5_h3_color_valence_contingency_memory$var == "Struan"],
                          results_exp_7_h3_color_valence_contingency_memory_excluded$logOR_se[results_exp_7_h3_color_valence_contingency_memory$var == "Struan"])) %>%
  mutate(yi  = logOR,
         sei = logOR_se) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2") %>%
  mutate(exclude = TRUE)

# hypothesis aware
results_for_meta_analysis_h3_hypothesis_aware_excluded <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", "Experiment 4", 
                            "Experiment 5", "Experiment 6", "Experiment 7"),
             logOR = c(results_exp_1_h3_hypothesis_aware_excluded$logOR[results_exp_1_h3_hypothesis_aware_excluded$var == "Struan"], 
                       results_exp_2_h3_hypothesis_aware_excluded$logOR[results_exp_2_h3_hypothesis_aware_excluded$var == "Struan"],
                       results_exp_3_h3_hypothesis_aware_excluded$logOR[results_exp_3_h3_hypothesis_aware_excluded$var == "Struan"],
                       results_exp_4_h3_hypothesis_aware_excluded$logOR[results_exp_4_h3_hypothesis_aware_excluded$var == "Struan"],
                       results_exp_5_h3_hypothesis_aware_excluded$logOR[results_exp_5_h3_hypothesis_aware_excluded$var == "Struan"],
                       results_exp_6_h3_hypothesis_aware_excluded$logOR[results_exp_6_h3_hypothesis_aware_excluded$var == "Struan"],
                       results_exp_7_h3_hypothesis_aware_excluded$logOR[results_exp_7_h3_hypothesis_aware_excluded$var == "Struan"]),
             logOR_se = c(results_exp_1_h3_hypothesis_aware_excluded$logOR_se[results_exp_1_h3_hypothesis_aware_excluded$var == "Struan"],
                          results_exp_2_h3_hypothesis_aware_excluded$logOR_se[results_exp_2_h3_hypothesis_aware_excluded$var == "Struan"],
                          results_exp_3_h3_hypothesis_aware_excluded$logOR_se[results_exp_3_h3_hypothesis_aware_excluded$var == "Struan"],
                          results_exp_4_h3_hypothesis_aware_excluded$logOR_se[results_exp_4_h3_hypothesis_aware_excluded$var == "Struan"],
                          results_exp_5_h3_hypothesis_aware_excluded$logOR_se[results_exp_5_h3_hypothesis_aware_excluded$var == "Struan"],
                          results_exp_6_h3_hypothesis_aware_excluded$logOR_se[results_exp_6_h3_hypothesis_aware_excluded$var == "Struan"],
                          results_exp_7_h3_hypothesis_aware_excluded$logOR_se[results_exp_7_h3_hypothesis_aware_excluded$var == "Struan"])) %>%
  mutate(yi  = logOR,
         sei = logOR_se) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2") %>%
  mutate(exclude = TRUE)


# meta fits
mod_meta_results_h3_us_contingency_memory <- 
  moderator_meta_analysis_workflow(data = bind_rows(semi_join(mutate(results_for_meta_analysis_h3_us_contingency_memory, exclude = FALSE),
                                                              results_for_meta_analysis_h3_us_contingency_memory_excluded,
                                                              by = "Experiment"),
                                                    results_for_meta_analysis_h3_us_contingency_memory_excluded),
                                   effect_size_label = "Odds Ratio")

mod_meta_results_h3_color_valence_contingency_memory <- 
  moderator_meta_analysis_workflow(data = bind_rows(semi_join(mutate(results_for_meta_analysis_h3_color_valence_contingency_memory, exclude = FALSE),
                                                              results_for_meta_analysis_h3_color_valence_contingency_memory_excluded,
                                                              by = "Experiment"),
                                                    results_for_meta_analysis_h3_color_valence_contingency_memory_excluded),
                                   effect_size_label = "Odds Ratio")

mod_meta_results_h3_hypothesis_aware <- 
  moderator_meta_analysis_workflow(data = bind_rows(semi_join(mutate(results_for_meta_analysis_h3_hypothesis_aware, exclude = FALSE),
                                                              results_for_meta_analysis_h3_hypothesis_aware_excluded,
                                                              by = "Experiment"),
                                                    results_for_meta_analysis_h3_hypothesis_aware_excluded),
                                   effect_size_label = "Odds Ratio")

# forest plots
forest(mod_meta_results_h3_us_contingency_memory$fitted_model)

forest(mod_meta_results_h3_color_valence_contingency_memory$fitted_model)

forest(mod_meta_results_h3_hypothesis_aware$fitted_model) 

Moderation by:

• US contingency memory (TRUE were excluded): QM(df = 1) = 5.24, p = 0.022, difference in excluded subset Odds Ratio = -0.92, 95% CI [-1.72, -0.13]
• color-valence contingency memory (TRUE were excluded): QM(df = 1) = 19.72, p < .001, difference in excluded subset Odds Ratio = -2.99, 95% CI [-4.32, -1.67]
• hypothesis awareness (FALSE were excluded): QM(df = 1) = 39.41, p < .001, difference in excluded subset Odds Ratio = 2.89, 95% CI [1.99, 3.79]